py/misc: Change sizeof to offsetof for variable-length alloc.

This fixes the case where e.g.

    struct foo_t {
      mp_obj_t x;
      uint16_t y;
      char buf[];
    };

will have `sizeof(struct foo_t)==8`, but `offsetof(struct foo_t, buf)==6`.

When computing the size to allocate for `m_new_obj_var` we need to use
offsetof to avoid over-allocating. This is important especially when it
might cause it to spill over into another GC block.

This work was funded through GitHub Sponsors.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
This commit is contained in:
Jim Mussared 2023-11-03 14:19:55 +11:00 committed by Damien George
parent c85db05244
commit b6a9778484
11 changed files with 24 additions and 24 deletions

View File

@ -207,12 +207,12 @@ STATIC mp_obj_t re_exec(bool is_anchored, uint n_args, const mp_obj_t *args) {
subj.begin_line = subj.begin = mp_obj_str_get_data(args[1], &len); subj.begin_line = subj.begin = mp_obj_str_get_data(args[1], &len);
subj.end = subj.begin + len; subj.end = subj.begin + len;
int caps_num = (self->re.sub + 1) * 2; int caps_num = (self->re.sub + 1) * 2;
mp_obj_match_t *match = m_new_obj_var(mp_obj_match_t, char *, caps_num); mp_obj_match_t *match = m_new_obj_var(mp_obj_match_t, caps, char *, caps_num);
// cast is a workaround for a bug in msvc: it treats const char** as a const pointer instead of a pointer to pointer to const char // cast is a workaround for a bug in msvc: it treats const char** as a const pointer instead of a pointer to pointer to const char
memset((char *)match->caps, 0, caps_num * sizeof(char *)); memset((char *)match->caps, 0, caps_num * sizeof(char *));
int res = re1_5_recursiveloopprog(&self->re, &subj, match->caps, caps_num, is_anchored); int res = re1_5_recursiveloopprog(&self->re, &subj, match->caps, caps_num, is_anchored);
if (res == 0) { if (res == 0) {
m_del_var(mp_obj_match_t, char *, caps_num, match); m_del_var(mp_obj_match_t, caps, char *, caps_num, match);
return mp_const_none; return mp_const_none;
} }

View File

@ -1701,7 +1701,7 @@ STATIC int create_l2cap_channel(uint16_t mtu, mp_bluetooth_nimble_l2cap_channel_
// multiply that by the "MTUs per channel" (set to 3 above). // multiply that by the "MTUs per channel" (set to 3 above).
const size_t buf_blocks = MP_CEIL_DIVIDE(mtu, L2CAP_BUF_BLOCK_SIZE) * L2CAP_BUF_SIZE_MTUS_PER_CHANNEL; const size_t buf_blocks = MP_CEIL_DIVIDE(mtu, L2CAP_BUF_BLOCK_SIZE) * L2CAP_BUF_SIZE_MTUS_PER_CHANNEL;
mp_bluetooth_nimble_l2cap_channel_t *chan = m_new_obj_var(mp_bluetooth_nimble_l2cap_channel_t, uint8_t, OS_MEMPOOL_SIZE(buf_blocks, L2CAP_BUF_BLOCK_SIZE) * sizeof(os_membuf_t)); mp_bluetooth_nimble_l2cap_channel_t *chan = m_new_obj_var(mp_bluetooth_nimble_l2cap_channel_t, sdu_mem, uint8_t, OS_MEMPOOL_SIZE(buf_blocks, L2CAP_BUF_BLOCK_SIZE) * sizeof(os_membuf_t));
MP_STATE_PORT(bluetooth_nimble_root_pointers)->l2cap_chan = chan; MP_STATE_PORT(bluetooth_nimble_root_pointers)->l2cap_chan = chan;
// Will be set in BLE_L2CAP_EVENT_COC_CONNECTED or BLE_L2CAP_EVENT_COC_ACCEPT. // Will be set in BLE_L2CAP_EVENT_COC_CONNECTED or BLE_L2CAP_EVENT_COC_ACCEPT.

View File

@ -90,9 +90,9 @@ mp_obj_t MP_VFS_LFSx(file_open)(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mod
} }
#if LFS_BUILD_VERSION == 1 #if LFS_BUILD_VERSION == 1
MP_OBJ_VFS_LFSx_FILE *o = m_new_obj_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, uint8_t, self->lfs.cfg->prog_size); MP_OBJ_VFS_LFSx_FILE *o = m_new_obj_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, file_buffer, uint8_t, self->lfs.cfg->prog_size);
#else #else
MP_OBJ_VFS_LFSx_FILE *o = m_new_obj_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, uint8_t, self->lfs.cfg->cache_size); MP_OBJ_VFS_LFSx_FILE *o = m_new_obj_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, file_buffer, uint8_t, self->lfs.cfg->cache_size);
#endif #endif
o->base.type = type; o->base.type = type;
o->vfs = self; o->vfs = self;

View File

@ -545,7 +545,7 @@ STATIC mp_obj_t extra_coverage(void) {
fun_bc.context = &context; fun_bc.context = &context;
fun_bc.child_table = NULL; fun_bc.child_table = NULL;
fun_bc.bytecode = (const byte *)"\x01"; // just needed for n_state fun_bc.bytecode = (const byte *)"\x01"; // just needed for n_state
mp_code_state_t *code_state = m_new_obj_var(mp_code_state_t, mp_obj_t, 1); mp_code_state_t *code_state = m_new_obj_var(mp_code_state_t, state, mp_obj_t, 1);
code_state->fun_bc = &fun_bc; code_state->fun_bc = &fun_bc;
code_state->ip = (const byte *)"\x00"; // just needed for an invalid opcode code_state->ip = (const byte *)"\x00"; // just needed for an invalid opcode
code_state->sp = &code_state->state[0]; code_state->sp = &code_state->state[0];

View File

@ -71,26 +71,26 @@ typedef unsigned int uint;
#define m_new0(type, num) ((type *)(m_malloc0(sizeof(type) * (num)))) #define m_new0(type, num) ((type *)(m_malloc0(sizeof(type) * (num))))
#define m_new_obj(type) (m_new(type, 1)) #define m_new_obj(type) (m_new(type, 1))
#define m_new_obj_maybe(type) (m_new_maybe(type, 1)) #define m_new_obj_maybe(type) (m_new_maybe(type, 1))
#define m_new_obj_var(obj_type, var_type, var_num) ((obj_type *)m_malloc(sizeof(obj_type) + sizeof(var_type) * (var_num))) #define m_new_obj_var(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num)))
#define m_new_obj_var0(obj_type, var_type, var_num) ((obj_type *)m_malloc0(sizeof(obj_type) + sizeof(var_type) * (var_num))) #define m_new_obj_var0(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc0(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num)))
#define m_new_obj_var_maybe(obj_type, var_type, var_num) ((obj_type *)m_malloc_maybe(sizeof(obj_type) + sizeof(var_type) * (var_num))) #define m_new_obj_var_maybe(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc_maybe(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num)))
#if MICROPY_ENABLE_FINALISER #if MICROPY_ENABLE_FINALISER
#define m_new_obj_with_finaliser(type) ((type *)(m_malloc_with_finaliser(sizeof(type)))) #define m_new_obj_with_finaliser(type) ((type *)(m_malloc_with_finaliser(sizeof(type))))
#define m_new_obj_var_with_finaliser(type, var_type, var_num) ((type *)m_malloc_with_finaliser(sizeof(type) + sizeof(var_type) * (var_num))) #define m_new_obj_var_with_finaliser(type, var_field, var_type, var_num) ((type *)m_malloc_with_finaliser(offsetof(type, var_field) + sizeof(var_type) * (var_num)))
#else #else
#define m_new_obj_with_finaliser(type) m_new_obj(type) #define m_new_obj_with_finaliser(type) m_new_obj(type)
#define m_new_obj_var_with_finaliser(type, var_type, var_num) m_new_obj_var(type, var_type, var_num) #define m_new_obj_var_with_finaliser(type, var_field, var_type, var_num) m_new_obj_var(type, var_field, var_type, var_num)
#endif #endif
#if MICROPY_MALLOC_USES_ALLOCATED_SIZE #if MICROPY_MALLOC_USES_ALLOCATED_SIZE
#define m_renew(type, ptr, old_num, new_num) ((type *)(m_realloc((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num)))) #define m_renew(type, ptr, old_num, new_num) ((type *)(m_realloc((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num))))
#define m_renew_maybe(type, ptr, old_num, new_num, allow_move) ((type *)(m_realloc_maybe((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num), (allow_move)))) #define m_renew_maybe(type, ptr, old_num, new_num, allow_move) ((type *)(m_realloc_maybe((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num), (allow_move))))
#define m_del(type, ptr, num) m_free(ptr, sizeof(type) * (num)) #define m_del(type, ptr, num) m_free(ptr, sizeof(type) * (num))
#define m_del_var(obj_type, var_type, var_num, ptr) (m_free(ptr, sizeof(obj_type) + sizeof(var_type) * (var_num))) #define m_del_var(obj_type, var_field, var_type, var_num, ptr) (m_free(ptr, offsetof(obj_type, var_field) + sizeof(var_type) * (var_num)))
#else #else
#define m_renew(type, ptr, old_num, new_num) ((type *)(m_realloc((ptr), sizeof(type) * (new_num)))) #define m_renew(type, ptr, old_num, new_num) ((type *)(m_realloc((ptr), sizeof(type) * (new_num))))
#define m_renew_maybe(type, ptr, old_num, new_num, allow_move) ((type *)(m_realloc_maybe((ptr), sizeof(type) * (new_num), (allow_move)))) #define m_renew_maybe(type, ptr, old_num, new_num, allow_move) ((type *)(m_realloc_maybe((ptr), sizeof(type) * (new_num), (allow_move))))
#define m_del(type, ptr, num) ((void)(num), m_free(ptr)) #define m_del(type, ptr, num) ((void)(num), m_free(ptr))
#define m_del_var(obj_type, var_type, var_num, ptr) ((void)(var_num), m_free(ptr)) #define m_del_var(obj_type, var_field, var_type, var_num, ptr) ((void)(var_num), m_free(ptr))
#endif #endif
#define m_del_obj(type, ptr) (m_del(type, ptr, 1)) #define m_del_obj(type, ptr) (m_del(type, ptr, 1))

View File

@ -235,7 +235,7 @@ STATIC mp_obj_t mod_thread_start_new_thread(size_t n_args, const mp_obj_t *args)
// check for keyword arguments // check for keyword arguments
if (n_args == 2) { if (n_args == 2) {
// just position arguments // just position arguments
th_args = m_new_obj_var(thread_entry_args_t, mp_obj_t, pos_args_len); th_args = m_new_obj_var(thread_entry_args_t, args, mp_obj_t, pos_args_len);
th_args->n_kw = 0; th_args->n_kw = 0;
} else { } else {
// positional and keyword arguments // positional and keyword arguments
@ -243,7 +243,7 @@ STATIC mp_obj_t mod_thread_start_new_thread(size_t n_args, const mp_obj_t *args)
mp_raise_TypeError(MP_ERROR_TEXT("expecting a dict for keyword args")); mp_raise_TypeError(MP_ERROR_TEXT("expecting a dict for keyword args"));
} }
mp_map_t *map = &((mp_obj_dict_t *)MP_OBJ_TO_PTR(args[2]))->map; mp_map_t *map = &((mp_obj_dict_t *)MP_OBJ_TO_PTR(args[2]))->map;
th_args = m_new_obj_var(thread_entry_args_t, mp_obj_t, pos_args_len + 2 * map->used); th_args = m_new_obj_var(thread_entry_args_t, args, mp_obj_t, pos_args_len + 2 * map->used);
th_args->n_kw = map->used; th_args->n_kw = map->used;
// copy across the keyword arguments // copy across the keyword arguments
for (size_t i = 0, n = pos_args_len; i < map->alloc; ++i) { for (size_t i = 0, n = pos_args_len; i < map->alloc; ++i) {

View File

@ -221,7 +221,7 @@ mp_obj_t mp_obj_exception_make_new(const mp_obj_type_t *type, size_t n_args, siz
o_tuple = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj; o_tuple = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
} else { } else {
// Try to allocate memory for the tuple containing the args // Try to allocate memory for the tuple containing the args
o_tuple = m_new_obj_var_maybe(mp_obj_tuple_t, mp_obj_t, n_args); o_tuple = m_new_obj_var_maybe(mp_obj_tuple_t, items, mp_obj_t, n_args);
#if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF
// If we are called by mp_obj_new_exception_msg_varg then it will have // If we are called by mp_obj_new_exception_msg_varg then it will have

View File

@ -215,7 +215,7 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args
// RuntimeError should be raised instead. So, we use m_new_obj_var_maybe(), // RuntimeError should be raised instead. So, we use m_new_obj_var_maybe(),
// return NULL, then vm.c takes the needed action (either raise // return NULL, then vm.c takes the needed action (either raise
// RuntimeError or fallback to stack allocation). // RuntimeError or fallback to stack allocation).
code_state = m_new_obj_var_maybe(mp_code_state_t, byte, state_size); code_state = m_new_obj_var_maybe(mp_code_state_t, state, byte, state_size);
if (!code_state) { if (!code_state) {
return NULL; return NULL;
} }
@ -247,10 +247,10 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const
// allocate state for locals and stack // allocate state for locals and stack
mp_code_state_t *code_state = NULL; mp_code_state_t *code_state = NULL;
#if MICROPY_ENABLE_PYSTACK #if MICROPY_ENABLE_PYSTACK
code_state = mp_pystack_alloc(sizeof(mp_code_state_t) + state_size); code_state = mp_pystack_alloc(offsetof(mp_code_state_t, state) + state_size);
#else #else
if (state_size > VM_MAX_STATE_ON_STACK) { if (state_size > VM_MAX_STATE_ON_STACK) {
code_state = m_new_obj_var_maybe(mp_code_state_t, byte, state_size); code_state = m_new_obj_var_maybe(mp_code_state_t, state, byte, state_size);
#if MICROPY_DEBUG_VM_STACK_OVERFLOW #if MICROPY_DEBUG_VM_STACK_OVERFLOW
if (code_state != NULL) { if (code_state != NULL) {
memset(code_state->state, 0, state_size); memset(code_state->state, 0, state_size);
@ -258,7 +258,7 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const
#endif #endif
} }
if (code_state == NULL) { if (code_state == NULL) {
code_state = alloca(sizeof(mp_code_state_t) + state_size); code_state = alloca(offsetof(mp_code_state_t, state) + state_size);
#if MICROPY_DEBUG_VM_STACK_OVERFLOW #if MICROPY_DEBUG_VM_STACK_OVERFLOW
memset(code_state->state, 0, state_size); memset(code_state->state, 0, state_size);
#endif #endif
@ -320,7 +320,7 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const
#else #else
// free the state if it was allocated on the heap // free the state if it was allocated on the heap
if (state_size != 0) { if (state_size != 0) {
m_del_var(mp_code_state_t, byte, state_size, code_state); m_del_var(mp_code_state_t, state, byte, state_size, code_state);
} }
#endif #endif

View File

@ -143,7 +143,7 @@ STATIC mp_obj_t namedtuple_make_new(const mp_obj_type_t *type_in, size_t n_args,
} }
mp_obj_namedtuple_type_t *mp_obj_new_namedtuple_base(size_t n_fields, mp_obj_t *fields) { mp_obj_namedtuple_type_t *mp_obj_new_namedtuple_base(size_t n_fields, mp_obj_t *fields) {
mp_obj_namedtuple_type_t *o = m_new_obj_var0(mp_obj_namedtuple_type_t, qstr, n_fields); mp_obj_namedtuple_type_t *o = m_new_obj_var0(mp_obj_namedtuple_type_t, fields, qstr, n_fields);
o->n_fields = n_fields; o->n_fields = n_fields;
for (size_t i = 0; i < n_fields; i++) { for (size_t i = 0; i < n_fields; i++) {
o->fields[i] = mp_obj_str_get_qstr(fields[i]); o->fields[i] = mp_obj_str_get_qstr(fields[i]);

View File

@ -264,7 +264,7 @@ void mp_obj_tuple_get(mp_obj_t self_in, size_t *len, mp_obj_t **items) {
void mp_obj_tuple_del(mp_obj_t self_in) { void mp_obj_tuple_del(mp_obj_t self_in) {
assert(mp_obj_is_type(self_in, &mp_type_tuple)); assert(mp_obj_is_type(self_in, &mp_type_tuple));
mp_obj_tuple_t *self = MP_OBJ_TO_PTR(self_in); mp_obj_tuple_t *self = MP_OBJ_TO_PTR(self_in);
m_del_var(mp_obj_tuple_t, mp_obj_t, self->len, self); m_del_var(mp_obj_tuple_t, items, mp_obj_t, self->len, self);
} }
/******************************************************************************/ /******************************************************************************/

View File

@ -1155,7 +1155,7 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict)
// (currently 10, plus 1 for base, plus 1 for base-protocol). // (currently 10, plus 1 for base, plus 1 for base-protocol).
// Note: mp_obj_type_t is (2 + 3 + #slots) words, so going from 11 to 12 slots // Note: mp_obj_type_t is (2 + 3 + #slots) words, so going from 11 to 12 slots
// moves from 4 to 5 gc blocks. // moves from 4 to 5 gc blocks.
mp_obj_type_t *o = m_new_obj_var0(mp_obj_type_t, void *, 10 + (bases_len ? 1 : 0) + (base_protocol ? 1 : 0)); mp_obj_type_t *o = m_new_obj_var0(mp_obj_type_t, slots, void *, 10 + (bases_len ? 1 : 0) + (base_protocol ? 1 : 0));
o->base.type = &mp_type_type; o->base.type = &mp_type_type;
o->flags = base_flags; o->flags = base_flags;
o->name = name; o->name = name;