From 42297811be2d1e7beb051b899c68cbce7837aec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1niel=20Buga?= Date: Wed, 17 Sep 2025 16:03:32 +0200 Subject: [PATCH] Use TLSF by default (#4130) * Use TLSF by default * Update example heap usage to leave more stack * Fix test configurations --- esp-alloc/CHANGELOG.md | 2 + esp-alloc/Cargo.toml | 7 +- esp-alloc/build.rs | 9 ++ esp-alloc/esp_config.yml | 14 +++ esp-alloc/src/heap/llff.rs | 40 ++++++++ esp-alloc/src/heap/mod.rs | 9 ++ esp-alloc/src/heap/tlsf.rs | 69 +++++++++++++ esp-alloc/src/lib.rs | 98 ++++++++++--------- examples/wifi/80211_tx/src/main.rs | 3 +- examples/wifi/access_point/src/main.rs | 3 +- .../wifi/access_point_with_sta/src/main.rs | 3 +- examples/wifi/dhcp/src/main.rs | 3 +- .../wifi/embassy_access_point/src/main.rs | 3 +- .../embassy_access_point_with_sta/src/main.rs | 3 +- examples/wifi/embassy_dhcp/src/main.rs | 3 +- hil-test/tests/aes.rs | 1 + hil-test/tests/alloc_psram.rs | 63 +++++++----- 17 files changed, 254 insertions(+), 79 deletions(-) create mode 100644 esp-alloc/build.rs create mode 100644 esp-alloc/esp_config.yml create mode 100644 esp-alloc/src/heap/llff.rs create mode 100644 esp-alloc/src/heap/mod.rs create mode 100644 esp-alloc/src/heap/tlsf.rs diff --git a/esp-alloc/CHANGELOG.md b/esp-alloc/CHANGELOG.md index 70d25ca6e..031ae7553 100644 --- a/esp-alloc/CHANGELOG.md +++ b/esp-alloc/CHANGELOG.md @@ -11,10 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added chip-selection features (#4023) - New default feature (`compat`) enables implementations for `malloc`, `free`, `calloc`, `realloc` and others (#3890, #4043) +- `ESP_ALLOC_CONFIG_HEAP_ALGORITHM` to select the global heap algorithm (#4130) ### Changed - Make stats structs fields public (#3828) +- The default heap allocator is now TLSF, implemented by the `rlsf` crate (#3950) ### Fixed diff --git a/esp-alloc/Cargo.toml b/esp-alloc/Cargo.toml index a8cb3595b..c4260be22 100644 --- a/esp-alloc/Cargo.toml +++ b/esp-alloc/Cargo.toml @@ -24,9 +24,14 @@ defmt = { version = "1.0.1", optional = true } cfg-if = "1.0.0" enumset = "1.1.6" esp-sync = { version = "0.0.0", path = "../esp-sync" } -linked_list_allocator = { version = "0.10.5", default-features = false, features = ["const_mut_refs"] } document-features = "0.2.11" +linked_list_allocator = { version = "0.10.5", default-features = false, features = ["const_mut_refs"] } +rlsf = { version = "0.2", features = ["unstable"] } + +[build-dependencies] +esp-config = { version = "0.5.0", path = "../esp-config", features = ["build"] } + [features] default = ["compat"] diff --git a/esp-alloc/build.rs b/esp-alloc/build.rs new file mode 100644 index 000000000..84dc1209a --- /dev/null +++ b/esp-alloc/build.rs @@ -0,0 +1,9 @@ +use esp_config::generate_config_from_yaml_definition; + +fn main() { + // emit config + println!("cargo:rerun-if-changed=./esp_config.yml"); + let cfg_yaml = std::fs::read_to_string("./esp_config.yml") + .expect("Failed to read esp_config.yml for esp-alloc"); + generate_config_from_yaml_definition(&cfg_yaml, true, true, None).unwrap(); +} diff --git a/esp-alloc/esp_config.yml b/esp-alloc/esp_config.yml new file mode 100644 index 000000000..6118bfe68 --- /dev/null +++ b/esp-alloc/esp_config.yml @@ -0,0 +1,14 @@ +crate: esp-alloc + +options: + - name: heap_algorithm + description: "The heap algorithm to use. TLSF offers higher performance + and bounded allocation time, but uses more memory." + default: + - value: '"TLSF"' + constraints: + - type: + validator: enumeration + value: + - "LLFF" + - "TLSF" diff --git a/esp-alloc/src/heap/llff.rs b/esp-alloc/src/heap/llff.rs new file mode 100644 index 000000000..b6c5fa39b --- /dev/null +++ b/esp-alloc/src/heap/llff.rs @@ -0,0 +1,40 @@ +use core::{alloc::Layout, ptr::NonNull}; + +use linked_list_allocator::Heap; + +pub(crate) struct LlffHeap { + heap: Heap, +} + +impl LlffHeap { + pub unsafe fn new(heap_bottom: *mut u8, size: usize) -> Self { + let mut heap = Heap::empty(); + unsafe { heap.init(heap_bottom, size) }; + Self { heap } + } + + pub fn size(&self) -> usize { + self.heap.size() + } + + pub fn used(&self) -> usize { + self.heap.used() + } + + pub fn free(&self) -> usize { + self.heap.free() + } + + pub fn allocate(&mut self, layout: Layout) -> Option> { + self.heap.allocate_first_fit(layout).ok() + } + + pub(crate) unsafe fn try_deallocate(&mut self, ptr: NonNull, layout: Layout) -> bool { + if self.heap.bottom() <= ptr.as_ptr() && self.heap.top() >= ptr.as_ptr() { + unsafe { self.heap.deallocate(ptr, layout) }; + true + } else { + false + } + } +} diff --git a/esp-alloc/src/heap/mod.rs b/esp-alloc/src/heap/mod.rs new file mode 100644 index 000000000..9bce65f0b --- /dev/null +++ b/esp-alloc/src/heap/mod.rs @@ -0,0 +1,9 @@ +#[cfg(heap_algorithm_llff)] +mod llff; +#[cfg(heap_algorithm_tlsf)] +mod tlsf; + +#[cfg(heap_algorithm_llff)] +pub(crate) use llff::LlffHeap as Heap; +#[cfg(heap_algorithm_tlsf)] +pub(crate) use tlsf::TlsfHeap as Heap; diff --git a/esp-alloc/src/heap/tlsf.rs b/esp-alloc/src/heap/tlsf.rs new file mode 100644 index 000000000..fae38bcab --- /dev/null +++ b/esp-alloc/src/heap/tlsf.rs @@ -0,0 +1,69 @@ +use core::{alloc::Layout, ptr::NonNull}; + +use rlsf::Tlsf; + +// TODO: make this configurable +type Heap = Tlsf<'static, usize, usize, { usize::BITS as usize }, { usize::BITS as usize }>; + +pub(crate) struct TlsfHeap { + heap: Heap, + pool_start: usize, + pool_end: usize, +} + +impl TlsfHeap { + pub unsafe fn new(heap_bottom: *mut u8, size: usize) -> Self { + let mut heap = Heap::new(); + + let block = unsafe { core::slice::from_raw_parts(heap_bottom, size) }; + let actual_size = unsafe { heap.insert_free_block_ptr(block.into()).unwrap() }; + + Self { + heap, + pool_start: heap_bottom as usize, + pool_end: heap_bottom as usize + actual_size.get(), + } + } + + pub fn size(&self) -> usize { + self.pool_end - self.pool_start + } + + pub fn used(&self) -> usize { + let mut used = 0; + let pool = + unsafe { core::slice::from_raw_parts(self.pool_start as *const u8, self.size()) }; + for block in unsafe { self.heap.iter_blocks(NonNull::from(pool)) } { + if block.is_occupied() { + used += block.size(); + } + } + used + } + + pub fn free(&self) -> usize { + let mut free = 0; + let pool = + unsafe { core::slice::from_raw_parts(self.pool_start as *const u8, self.size()) }; + for block in unsafe { self.heap.iter_blocks(NonNull::from(pool)) } { + if !block.is_occupied() { + free += block.max_payload_size(); + } + } + free + } + + pub fn allocate(&mut self, layout: Layout) -> Option> { + self.heap.allocate(layout) + } + + pub(crate) unsafe fn try_deallocate(&mut self, ptr: NonNull, layout: Layout) -> bool { + let addr = ptr.addr().get(); + if self.pool_start <= addr && self.pool_end > addr { + unsafe { self.heap.deallocate(ptr, layout.align()) }; + true + } else { + false + } + } +} diff --git a/esp-alloc/src/lib.rs b/esp-alloc/src/lib.rs index b0fee7009..e61879017 100644 --- a/esp-alloc/src/lib.rs +++ b/esp-alloc/src/lib.rs @@ -143,6 +143,7 @@ #![doc(html_logo_url = "https://avatars.githubusercontent.com/u/46717278")] mod allocators; +mod heap; mod macros; #[cfg(feature = "compat")] mod malloc; @@ -156,7 +157,8 @@ use core::{ pub use allocators::*; use enumset::{EnumSet, EnumSetType}; use esp_sync::NonReentrantMutex; -use linked_list_allocator::Heap; + +use crate::heap::Heap; /// The global allocator instance #[global_allocator] @@ -275,23 +277,41 @@ impl HeapRegion { size: usize, capabilities: EnumSet, ) -> Self { - unsafe { - let mut heap = Heap::empty(); - heap.init(heap_bottom, size); - - Self { heap, capabilities } + Self { + heap: unsafe { Heap::new(heap_bottom, size) }, + capabilities, } } /// Return stats for the current memory region pub fn stats(&self) -> RegionStats { RegionStats { - size: self.heap.size(), - used: self.heap.used(), - free: self.heap.free(), + size: self.size(), + used: self.used(), + free: self.free(), capabilities: self.capabilities, } } + + fn size(&self) -> usize { + self.heap.size() + } + + fn used(&self) -> usize { + self.heap.used() + } + + fn free(&self) -> usize { + self.heap.free() + } + + fn allocate(&mut self, layout: Layout) -> Option> { + self.heap.allocate(layout) + } + + unsafe fn try_deallocate(&mut self, ptr: NonNull, layout: Layout) -> bool { + unsafe { self.heap.try_deallocate(ptr, layout) } + } } /// Stats for a heap allocator @@ -503,44 +523,34 @@ impl EspHeapInner { ) -> *mut u8 { #[cfg(feature = "internal-heap-stats")] let before = self.used(); + let mut iter = self + .heap + .iter_mut() + .filter_map(|region| region.as_mut()) + .filter(|region| region.capabilities.is_superset(capabilities)); - let mut iter = self.heap.iter_mut().filter(|region| { - if region.is_some() { - region - .as_ref() - .unwrap() - .capabilities - .is_superset(capabilities) - } else { - false - } - }); + let allocation = loop { + let Some(region) = iter.next() else { + return ptr::null_mut(); + }; - let res = loop { - if let Some(Some(region)) = iter.next() { - let res = region.heap.allocate_first_fit(layout); - if let Ok(res) = res { - break Some(res); - } - } else { - break None; + if let Some(res) = region.allocate(layout) { + break res; } }; - res.map_or(ptr::null_mut(), |allocation| { - #[cfg(feature = "internal-heap-stats")] - { - // We need to call used because [linked_list_allocator::Heap] does internal size - // alignment so we cannot use the size provided by the layout. - let used = self.used(); + #[cfg(feature = "internal-heap-stats")] + { + // We need to call used because the heap impls have some internal overhead + // so we cannot use the size provided by the layout. + let used = self.used(); - self.internal_heap_stats.total_allocated += used - before; - self.internal_heap_stats.max_usage = - core::cmp::max(self.internal_heap_stats.max_usage, used); - } + self.internal_heap_stats.total_allocated += used - before; + self.internal_heap_stats.max_usage = + core::cmp::max(self.internal_heap_stats.max_usage, used); + } - allocation.as_ptr() - }) + allocation.as_ptr() } } @@ -637,9 +647,9 @@ unsafe impl GlobalAlloc for EspHeap { } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - if ptr.is_null() { + let Some(ptr) = NonNull::new(ptr) else { return; - } + }; self.inner.with(|this| { #[cfg(feature = "internal-heap-stats")] @@ -647,8 +657,8 @@ unsafe impl GlobalAlloc for EspHeap { let mut iter = this.heap.iter_mut(); while let Some(Some(region)) = iter.next() { - if region.heap.bottom() <= ptr && region.heap.top() >= ptr { - unsafe { region.heap.deallocate(NonNull::new_unchecked(ptr), layout) }; + if unsafe { region.try_deallocate(ptr, layout) } { + break; } } diff --git a/examples/wifi/80211_tx/src/main.rs b/examples/wifi/80211_tx/src/main.rs index 56118c4ce..6d00a283e 100644 --- a/examples/wifi/80211_tx/src/main.rs +++ b/examples/wifi/80211_tx/src/main.rs @@ -33,7 +33,8 @@ fn main() -> ! { let config = esp_hal::Config::default().with_cpu_clock(CpuClock::max()); let peripherals = esp_hal::init(config); - esp_alloc::heap_allocator!(size: 72 * 1024); + esp_alloc::heap_allocator!(#[unsafe(link_section = ".dram2_uninit")] size: 64 * 1024); + esp_alloc::heap_allocator!(size: 36 * 1024); let delay = Delay::new(); diff --git a/examples/wifi/access_point/src/main.rs b/examples/wifi/access_point/src/main.rs index 51e26ee9d..eef0f03b2 100644 --- a/examples/wifi/access_point/src/main.rs +++ b/examples/wifi/access_point/src/main.rs @@ -40,7 +40,8 @@ fn main() -> ! { let config = esp_hal::Config::default().with_cpu_clock(CpuClock::max()); let peripherals = esp_hal::init(config); - esp_alloc::heap_allocator!(size: 72 * 1024); + esp_alloc::heap_allocator!(#[unsafe(link_section = ".dram2_uninit")] size: 64 * 1024); + esp_alloc::heap_allocator!(size: 36 * 1024); let timg0 = TimerGroup::new(peripherals.TIMG0); esp_preempt::start(timg0.timer0); diff --git a/examples/wifi/access_point_with_sta/src/main.rs b/examples/wifi/access_point_with_sta/src/main.rs index ec0808d14..9623e148a 100644 --- a/examples/wifi/access_point_with_sta/src/main.rs +++ b/examples/wifi/access_point_with_sta/src/main.rs @@ -44,7 +44,8 @@ fn main() -> ! { let config = esp_hal::Config::default().with_cpu_clock(CpuClock::max()); let peripherals = esp_hal::init(config); - esp_alloc::heap_allocator!(size: 72 * 1024); + esp_alloc::heap_allocator!(#[unsafe(link_section = ".dram2_uninit")] size: 64 * 1024); + esp_alloc::heap_allocator!(size: 36 * 1024); let timg0 = TimerGroup::new(peripherals.TIMG0); esp_preempt::start(timg0.timer0); diff --git a/examples/wifi/dhcp/src/main.rs b/examples/wifi/dhcp/src/main.rs index 7cc6f0e7f..b764fea84 100644 --- a/examples/wifi/dhcp/src/main.rs +++ b/examples/wifi/dhcp/src/main.rs @@ -41,7 +41,8 @@ fn main() -> ! { let config = esp_hal::Config::default().with_cpu_clock(CpuClock::max()); let peripherals = esp_hal::init(config); - esp_alloc::heap_allocator!(size: 72 * 1024); + esp_alloc::heap_allocator!(#[unsafe(link_section = ".dram2_uninit")] size: 64 * 1024); + esp_alloc::heap_allocator!(size: 36 * 1024); let timg0 = TimerGroup::new(peripherals.TIMG0); esp_preempt::start(timg0.timer0); diff --git a/examples/wifi/embassy_access_point/src/main.rs b/examples/wifi/embassy_access_point/src/main.rs index 7820165f8..a52340f47 100644 --- a/examples/wifi/embassy_access_point/src/main.rs +++ b/examples/wifi/embassy_access_point/src/main.rs @@ -55,7 +55,8 @@ async fn main(spawner: Spawner) -> ! { let config = esp_hal::Config::default().with_cpu_clock(CpuClock::max()); let peripherals = esp_hal::init(config); - esp_alloc::heap_allocator!(size: 72 * 1024); + esp_alloc::heap_allocator!(#[unsafe(link_section = ".dram2_uninit")] size: 64 * 1024); + esp_alloc::heap_allocator!(size: 36 * 1024); let timg0 = TimerGroup::new(peripherals.TIMG0); esp_preempt::start(timg0.timer0); diff --git a/examples/wifi/embassy_access_point_with_sta/src/main.rs b/examples/wifi/embassy_access_point_with_sta/src/main.rs index b552bbbeb..d19b018db 100644 --- a/examples/wifi/embassy_access_point_with_sta/src/main.rs +++ b/examples/wifi/embassy_access_point_with_sta/src/main.rs @@ -70,7 +70,8 @@ async fn main(spawner: Spawner) -> ! { let config = esp_hal::Config::default().with_cpu_clock(CpuClock::max()); let peripherals = esp_hal::init(config); - esp_alloc::heap_allocator!(size: 72 * 1024); + esp_alloc::heap_allocator!(#[unsafe(link_section = ".dram2_uninit")] size: 64 * 1024); + esp_alloc::heap_allocator!(size: 36 * 1024); let timg0 = TimerGroup::new(peripherals.TIMG0); esp_preempt::start(timg0.timer0); diff --git a/examples/wifi/embassy_dhcp/src/main.rs b/examples/wifi/embassy_dhcp/src/main.rs index 7fcd80718..b7dcd223a 100644 --- a/examples/wifi/embassy_dhcp/src/main.rs +++ b/examples/wifi/embassy_dhcp/src/main.rs @@ -46,7 +46,8 @@ async fn main(spawner: Spawner) -> ! { let config = esp_hal::Config::default().with_cpu_clock(CpuClock::max()); let peripherals = esp_hal::init(config); - esp_alloc::heap_allocator!(size: 72 * 1024); + esp_alloc::heap_allocator!(#[unsafe(link_section = ".dram2_uninit")] size: 64 * 1024); + esp_alloc::heap_allocator!(size: 36 * 1024); let timg0 = TimerGroup::new(peripherals.TIMG0); esp_preempt::start(timg0.timer0); diff --git a/hil-test/tests/aes.rs b/hil-test/tests/aes.rs index d592b917e..3f00ee0bf 100644 --- a/hil-test/tests/aes.rs +++ b/hil-test/tests/aes.rs @@ -326,6 +326,7 @@ fn run_cipher_tests(buffer: &mut [u8]) { ); } +#[cfg(aes_dma)] fn run_unaligned_dma_tests(memory: &mut [u8]) { let zeros = [0; MAX_SHIFT]; diff --git a/hil-test/tests/alloc_psram.rs b/hil-test/tests/alloc_psram.rs index 1d16a9715..b3ff39744 100644 --- a/hil-test/tests/alloc_psram.rs +++ b/hil-test/tests/alloc_psram.rs @@ -1,9 +1,12 @@ //! Allocator and PSRAM-related tests -//% CHIPS(quad): esp32 esp32s2 +//% CHIPS: +//% CHIPS(llff_quad, tlsf_quad): esp32 esp32s2 // The S3 dev kit in the HIL-tester has octal PSRAM. -//% CHIPS(octal): esp32s3 -//% ENV(octal): ESP_HAL_CONFIG_PSRAM_MODE=octal +//% CHIPS(llff_octal, tlsf_octal): esp32s3 +//% ENV(llff_octal, tlsf_octal): ESP_HAL_CONFIG_PSRAM_MODE=octal +//% ENV(llff_octal, llff_quad): ESP_ALLOC_CONFIG_HEAP_ALGORITHM=LLFF +//% ENV(tlsf_octal, tlsf_quad): ESP_ALLOC_CONFIG_HEAP_ALGORITHM=TLSF //% FEATURES: unstable psram esp-storage esp-alloc/nightly #![no_std] @@ -50,16 +53,18 @@ mod tests { #[test] fn all_psram_is_usable() { - let free = esp_alloc::HEAP.free(); - defmt::info!("Free: {}", free); - let mut vec = AllocVec::with_capacity(free); + if option_env!("ESP_ALLOC_CONFIG_HEAP_ALGORITHM") == Some("LLFF") { + let free = esp_alloc::HEAP.free(); + defmt::info!("Free: {}", free); + let mut vec = AllocVec::with_capacity(free); - for i in 0..free { - vec.push((i % 256) as u8); - } + for i in 0..free { + vec.push((i % 256) as u8); + } - for i in 0..free { - assert_eq!(vec[i], (i % 256) as u8); + for i in 0..free { + assert_eq!(vec[i], (i % 256) as u8); + } } } @@ -109,31 +114,35 @@ mod tests { #[test] fn all_psram_is_usable_with_external_mem_allocator() { - let free = esp_alloc::HEAP.free(); - defmt::info!("Free: {}", free); - let mut vec = Vec::with_capacity_in(free, ExternalMemory); + if option_env!("ESP_ALLOC_CONFIG_HEAP_ALGORITHM") == Some("LLFF") { + let free = esp_alloc::HEAP.free(); + defmt::info!("Free: {}", free); + let mut vec = Vec::with_capacity_in(free, ExternalMemory); - for i in 0..free { - vec.push((i % 256) as u8); - } + for i in 0..free { + vec.push((i % 256) as u8); + } - for i in 0..free { - assert_eq!(vec[i], (i % 256) as u8); + for i in 0..free { + assert_eq!(vec[i], (i % 256) as u8); + } } } #[test] fn all_psram_is_usable_with_any_mem_allocator() { - let free = esp_alloc::HEAP.free(); - defmt::info!("Free: {}", free); - let mut vec = Vec::with_capacity_in(free, AnyMemory); + if option_env!("ESP_ALLOC_CONFIG_HEAP_ALGORITHM") == Some("LLFF") { + let free = esp_alloc::HEAP.free(); + defmt::info!("Free: {}", free); + let mut vec = Vec::with_capacity_in(free, AnyMemory); - for i in 0..free { - vec.push((i % 256) as u8); - } + for i in 0..free { + vec.push((i % 256) as u8); + } - for i in 0..free { - assert_eq!(vec[i], (i % 256) as u8); + for i in 0..free { + assert_eq!(vec[i], (i % 256) as u8); + } } }