esp32: Enable automatic Python heap growth.

Via MICROPY_GC_SPLIT_HEAP_AUTO feature flag added in previous commit.

Tested on ESP32 GENERIC_SPIRAM and GENERIC_S3 configurations, with some
worst-case allocation patterns and the standard test suite.

This work was funded through GitHub Sponsors.

Signed-off-by: Angus Gratton <angus@redyak.com.au>
This commit is contained in:
Angus Gratton 2023-08-02 16:51:07 +10:00 committed by Damien George
parent 98fd78437c
commit 05dcb8be99
4 changed files with 28 additions and 10 deletions

View File

@ -51,13 +51,20 @@ Functions
buffers and other data. This data is useful to get a sense of how much memory buffers and other data. This data is useful to get a sense of how much memory
is available to ESP-IDF and the networking stack in particular. It may shed is available to ESP-IDF and the networking stack in particular. It may shed
some light on situations where ESP-IDF operations fail due to allocation failures. some light on situations where ESP-IDF operations fail due to allocation failures.
The information returned is *not* useful to troubleshoot Python allocation failures,
use `micropython.mem_info()` instead.
The capabilities parameter corresponds to ESP-IDF's ``MALLOC_CAP_XXX`` values but the The capabilities parameter corresponds to ESP-IDF's ``MALLOC_CAP_XXX`` values but the
two most useful ones are predefined as `esp32.HEAP_DATA` for data heap regions and two most useful ones are predefined as `esp32.HEAP_DATA` for data heap regions and
`esp32.HEAP_EXEC` for executable regions as used by the native code emitter. `esp32.HEAP_EXEC` for executable regions as used by the native code emitter.
Free IDF heap memory in the `esp32.HEAP_DATA` region is available to be
automatically added to the MicroPython heap to prevent a MicroPython
allocation from failing. However, the information returned here is otherwise
*not* useful to troubleshoot Python allocation failures, use
`micropython.mem_info()` instead. The "max new split" value in
`micropython.mem_info()` output corresponds to the largest free block of
ESP-IDF heap that could be automatically added on demand to the MicroPython
heap.
The return value is a list of 4-tuples, where each 4-tuple corresponds to one heap The return value is a list of 4-tuples, where each 4-tuple corresponds to one heap
and contains: the total bytes, the free bytes, the largest free block, and and contains: the total bytes, the free bytes, the largest free block, and
the minimum free seen over time. the minimum free seen over time.

View File

@ -80,3 +80,13 @@ void gc_collect(void) {
} }
#endif #endif
#if MICROPY_GC_SPLIT_HEAP_AUTO
// The largest new region that is available to become Python heap is the largest
// free block in the ESP-IDF system heap.
size_t gc_get_max_new_split(void) {
return heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT);
}
#endif

View File

@ -75,6 +75,10 @@
#define MP_TASK_STACK_LIMIT_MARGIN (1024) #define MP_TASK_STACK_LIMIT_MARGIN (1024)
#endif #endif
// Initial Python heap size. This starts small but adds new heap areas on
// demand due to settings MICROPY_GC_SPLIT_HEAP & MICROPY_GC_SPLIT_HEAP_AUTO
#define MP_TASK_HEAP_SIZE (64 * 1024)
int vprintf_null(const char *format, va_list ap) { int vprintf_null(const char *format, va_list ap) {
// do nothing: this is used as a log target during raw repl mode // do nothing: this is used as a log target during raw repl mode
return 0; return 0;
@ -100,19 +104,13 @@ void mp_task(void *pvParameter) {
ESP_LOGE("esp_init", "can't create event loop: 0x%x\n", err); ESP_LOGE("esp_init", "can't create event loop: 0x%x\n", err);
} }
// Allocate the uPy heap using malloc and get the largest available region, void *mp_task_heap = MP_PLAT_ALLOC_HEAP(MP_TASK_HEAP_SIZE);
// limiting to 1/2 total available memory to leave memory for the OS.
// When SPIRAM is enabled, this will allocate from SPIRAM.
uint32_t caps = MALLOC_CAP_8BIT;
size_t heap_total = heap_caps_get_total_size(caps);
size_t mp_task_heap_size = MIN(heap_caps_get_largest_free_block(caps), heap_total / 2);
void *mp_task_heap = heap_caps_malloc(mp_task_heap_size, caps);
soft_reset: soft_reset:
// initialise the stack pointer for the main thread // initialise the stack pointer for the main thread
mp_stack_set_top((void *)sp); mp_stack_set_top((void *)sp);
mp_stack_set_limit(MP_TASK_STACK_SIZE - MP_TASK_STACK_LIMIT_MARGIN); mp_stack_set_limit(MP_TASK_STACK_SIZE - MP_TASK_STACK_LIMIT_MARGIN);
gc_init(mp_task_heap, mp_task_heap + mp_task_heap_size); gc_init(mp_task_heap, mp_task_heap + MP_TASK_HEAP_SIZE);
mp_init(); mp_init();
mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib));
readline_init0(); readline_init0();

View File

@ -68,6 +68,9 @@
#define MICROPY_PY_THREAD_GIL (1) #define MICROPY_PY_THREAD_GIL (1)
#define MICROPY_PY_THREAD_GIL_VM_DIVISOR (32) #define MICROPY_PY_THREAD_GIL_VM_DIVISOR (32)
#define MICROPY_GC_SPLIT_HEAP (1)
#define MICROPY_GC_SPLIT_HEAP_AUTO (1)
// extended modules // extended modules
#ifndef MICROPY_ESPNOW #ifndef MICROPY_ESPNOW
#define MICROPY_ESPNOW (1) #define MICROPY_ESPNOW (1)