From 58c9586c343870e721512975af55cf210b45f756 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 12 Jul 2014 14:51:48 +0300 Subject: [PATCH 1/8] emitbc: Fix structure field alignment issue. dummy_data field is accessed as uint value (e.g. in emit_write_bytecode_byte_ptr), but is not aligned as such, which causes bus errors or incorrect behavior on any arch requiring strictly aligned data (ARM pre-v7, MIPS, etc, etc). --- py/emitbc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/py/emitbc.c b/py/emitbc.c index ebc2ba500..365ec458a 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -50,7 +50,6 @@ struct _emit_t { pass_kind_t pass : 8; uint last_emit_was_return_value : 8; - byte dummy_data[DUMMY_DATA_SIZE]; int stack_size; @@ -67,6 +66,8 @@ struct _emit_t { uint bytecode_offset; uint bytecode_size; byte *code_base; // stores both byte code and code info + // Accessed as uint, so must be aligned as such + byte dummy_data[DUMMY_DATA_SIZE]; }; STATIC void emit_bc_rot_two(emit_t *emit); @@ -207,6 +208,8 @@ STATIC void emit_write_bytecode_byte_ptr(emit_t* emit, byte b, void *ptr) { emit_write_bytecode_byte(emit, b); emit_align_bytecode_to_machine_word(emit); mp_uint_t *c = (mp_uint_t*)emit_get_cur_to_write_bytecode(emit, sizeof(mp_uint_t)); + // Verify thar c is already uint-aligned + assert(c == MP_ALIGN(c, sizeof(mp_uint_t))); *c = (mp_uint_t)ptr; } From 564e46452db09d603e3c0168fbaa008a5202d431 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 12 Jul 2014 15:55:47 +0300 Subject: [PATCH 2/8] py: Add generic helper to align a pointer. --- py/misc.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/py/misc.h b/py/misc.h index c8ccdaa02..94a3ffaaf 100644 --- a/py/misc.h +++ b/py/misc.h @@ -84,6 +84,9 @@ int m_get_peak_bytes_allocated(void); // get the number of elements in a fixed-size array #define MP_ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +// align ptr to the nearest multiple of "alignment" +#define MP_ALIGN(ptr, alignment) (void*)(((mp_uint_t)(ptr) + ((alignment) - 1)) & ~((alignment) - 1)) + /** unichar / UTF-8 *********************************************/ typedef int unichar; // TODO From 2cf381081ae0b7c3fa80814bebd626cb678ca766 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 12 Jul 2014 16:34:51 +0300 Subject: [PATCH 3/8] run-tests: Add option to write CPython's test results to .exp files. Mostly to run testsuite on targets which doesn't have CPython. --- tests/run-tests | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index c6bc4020d..71a94f946 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -20,7 +20,7 @@ def rm_f(fname): if os.path.exists(fname): os.remove(fname) -def run_tests(pyb, tests): +def run_tests(pyb, tests, args): test_count = 0 testcase_count = 0 passed_count = 0 @@ -54,9 +54,15 @@ def run_tests(pyb, tests): # run CPython to work out expected output try: output_expected = subprocess.check_output([CPYTHON3, '-B', test_file]) + if args.write_exp: + with open(test_file_expected, 'wb') as f: + f.write(output_expected) except subprocess.CalledProcessError: output_expected = b'CPYTHON3 CRASH' + if args.write_exp: + continue + # run Micro Python if pyb is None: # run on PC @@ -113,6 +119,7 @@ def main(): cmd_parser = argparse.ArgumentParser(description='Run tests for Micro Python.') cmd_parser.add_argument('--pyboard', action='store_true', help='run the tests on the pyboard') cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)') + cmd_parser.add_argument('--write-exp', action='store_true', help='save .exp files to run tests w/o CPython') cmd_parser.add_argument('files', nargs='*', help='input test files') args = cmd_parser.parse_args() @@ -139,7 +146,7 @@ def main(): # tests explicitly given tests = args.files - if not run_tests(pyb, tests): + if not run_tests(pyb, tests, args): sys.exit(1) if __name__ == "__main__": From b82f34edde79e14d6b9260b3fddaf36b1354a558 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 13 Jul 2014 13:49:51 +0300 Subject: [PATCH 4/8] unix: Allow to disable MICROPY_EMIT_X64 from commandline. emitnative in particular requires nlr_* to be real functions, so doesn't compile with MICROPY_NLR_SETJMP=1. --- unix/mpconfigport.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unix/mpconfigport.h b/unix/mpconfigport.h index 73435863b..0831e3fd3 100644 --- a/unix/mpconfigport.h +++ b/unix/mpconfigport.h @@ -27,7 +27,9 @@ // options to control how Micro Python is built #define MICROPY_ALLOC_PATH_MAX (PATH_MAX) +#ifndef MICROPY_EMIT_X64 #define MICROPY_EMIT_X64 (1) +#endif #define MICROPY_EMIT_THUMB (0) #define MICROPY_EMIT_INLINE_THUMB (0) #define MICROPY_ENABLE_GC (1) From a1760a56ff2a9be9506e96f4cd17459bf2916b30 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 13 Jul 2014 18:49:56 +0300 Subject: [PATCH 5/8] test: Add run-tests-exp.sh, script to run testsuite with only sh dependency. This script uses expected test results as generated by run-tests --write-exp, and requires only standard unix shell funtionality (no bash). It is useful to run testsuite on embedded systems, where there's no CPython and Bash. --- tests/run-tests-exp.sh | 57 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100755 tests/run-tests-exp.sh diff --git a/tests/run-tests-exp.sh b/tests/run-tests-exp.sh new file mode 100755 index 000000000..032c42dd3 --- /dev/null +++ b/tests/run-tests-exp.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# +# This is plain shell variant of run-tests script, which uses .exp files +# as generated by run-tests --write-exp. It is useful to run testsuite +# on embedded systems which doesn't have CPython3. +# + +RM="rm -f" +MP_PY=micropython + +numtests=0 +numtestcases=0 +numpassed=0 +numfailed=0 +namefailed= + +if [ $# -eq 0 ] +then + tests="basics/*.py micropython/*.py float/*.py import/*.py io/*.py misc/*.py" +else + tests="$@" +fi + +for infile in $tests +do + basename=`basename $infile .py` + outfile=${basename}.out + expfile=$infile.exp + + $MP_PY $infile > $outfile + numtestcases=$(expr $numtestcases + $(cat $expfile | wc -l)) + + diff --brief $expfile $outfile > /dev/null + + if [ $? -eq 0 ] + then + echo "pass $infile" + $RM $outfile + numpassed=$(expr $numpassed + 1) + else + echo "FAIL $infile" + numfailed=$(expr $numfailed + 1) + namefailed="$namefailed $basename" + fi + + numtests=$(expr $numtests + 1) +done + +echo "$numtests tests performed ($numtestcases individual testcases)" +echo "$numpassed tests passed" +if [ $numfailed != 0 ] +then + echo "$numfailed tests failed -$namefailed" + exit 1 +else + exit 0 +fi From 122c9db3dbbb6286c81d59cc979799d34974cae0 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 13 Jul 2014 23:05:24 +0300 Subject: [PATCH 6/8] unix: file: Implement .flush() method. This method apparently should be part of stream interface. --- unix/file.c | 8 ++++++++ unix/qstrdefsport.h | 1 + 2 files changed, 9 insertions(+) diff --git a/unix/file.c b/unix/file.c index 62e3253b7..433decfc3 100644 --- a/unix/file.c +++ b/unix/file.c @@ -82,6 +82,13 @@ STATIC mp_int_t fdfile_write(mp_obj_t o_in, const void *buf, mp_uint_t size, int return r; } +STATIC mp_obj_t fdfile_flush(mp_obj_t self_in) { + mp_obj_fdfile_t *self = self_in; + fsync(self->fd); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(fdfile_flush_obj, fdfile_flush); + STATIC mp_obj_t fdfile_close(mp_obj_t self_in) { mp_obj_fdfile_t *self = self_in; close(self->fd); @@ -166,6 +173,7 @@ STATIC const mp_map_elem_t rawfile_locals_dict_table[] = { { MP_OBJ_NEW_QSTR(MP_QSTR_readline), (mp_obj_t)&mp_stream_unbuffered_readline_obj}, { MP_OBJ_NEW_QSTR(MP_QSTR_readlines), (mp_obj_t)&mp_stream_unbuffered_readlines_obj}, { MP_OBJ_NEW_QSTR(MP_QSTR_write), (mp_obj_t)&mp_stream_write_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_flush), (mp_obj_t)&fdfile_flush_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_close), (mp_obj_t)&fdfile_close_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR___enter__), (mp_obj_t)&mp_identity_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR___exit__), (mp_obj_t)&fdfile___exit___obj }, diff --git a/unix/qstrdefsport.h b/unix/qstrdefsport.h index de0b3d8fa..05c918017 100644 --- a/unix/qstrdefsport.h +++ b/unix/qstrdefsport.h @@ -32,6 +32,7 @@ Q(fileno) Q(makefile) Q(FileIO) +Q(flush) Q(_os) Q(stat) From ac736f15c995466a0852506438d95934d65d2179 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 13 Jul 2014 23:04:17 +0300 Subject: [PATCH 7/8] stream: Factor out mp_stream_write() method to write a memstring to stream. --- py/stream.c | 15 +++++++++------ py/stream.h | 2 ++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/py/stream.c b/py/stream.c index 98f62518a..2b4410728 100644 --- a/py/stream.c +++ b/py/stream.c @@ -98,18 +98,15 @@ STATIC mp_obj_t stream_read(uint n_args, const mp_obj_t *args) { } } -STATIC mp_obj_t stream_write(mp_obj_t self_in, mp_obj_t arg) { +mp_obj_t mp_stream_write(mp_obj_t self_in, const void *buf, mp_uint_t len) { struct _mp_obj_base_t *o = (struct _mp_obj_base_t *)self_in; if (o->type->stream_p == NULL || o->type->stream_p->write == NULL) { // CPython: io.UnsupportedOperation, OSError subclass nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "Operation not supported")); } - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ); - int error; - mp_int_t out_sz = o->type->stream_p->write(self_in, bufinfo.buf, bufinfo.len, &error); + mp_int_t out_sz = o->type->stream_p->write(self_in, buf, len, &error); if (out_sz == -1) { if (is_nonblocking_error(error)) { // http://docs.python.org/3/library/io.html#io.RawIOBase.write @@ -125,6 +122,12 @@ STATIC mp_obj_t stream_write(mp_obj_t self_in, mp_obj_t arg) { } } +STATIC mp_obj_t stream_write_method(mp_obj_t self_in, mp_obj_t arg) { + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ); + return mp_stream_write(self_in, bufinfo.buf, bufinfo.len); +} + STATIC mp_obj_t stream_readall(mp_obj_t self_in) { struct _mp_obj_base_t *o = (struct _mp_obj_base_t *)self_in; if (o->type->stream_p == NULL || o->type->stream_p->read == NULL) { @@ -248,4 +251,4 @@ mp_obj_t mp_stream_unbuffered_iter(mp_obj_t self) { MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read_obj, 1, 2, stream_read); MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_readall_obj, stream_readall); MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj, 1, 2, stream_unbuffered_readline); -MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_write_obj, stream_write); +MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_write_obj, stream_write_method); diff --git a/py/stream.h b/py/stream.h index e52508daa..4cdc1b4dc 100644 --- a/py/stream.h +++ b/py/stream.h @@ -32,3 +32,5 @@ MP_DECLARE_CONST_FUN_OBJ(mp_stream_write_obj); // Iterator which uses mp_stream_unbuffered_readline_obj mp_obj_t mp_stream_unbuffered_iter(mp_obj_t self); + +mp_obj_t mp_stream_write(mp_obj_t self_in, const void *buf, mp_uint_t len); From dce8876dbe272d34d8d28aac21b4a4c3bdea0317 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 13 Jul 2014 23:24:08 +0300 Subject: [PATCH 8/8] unix: file: No fsync() on Windows. --- unix/file.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/unix/file.c b/unix/file.c index 433decfc3..056a7b6e8 100644 --- a/unix/file.c +++ b/unix/file.c @@ -83,8 +83,12 @@ STATIC mp_int_t fdfile_write(mp_obj_t o_in, const void *buf, mp_uint_t size, int } STATIC mp_obj_t fdfile_flush(mp_obj_t self_in) { +#ifndef _WIN32 mp_obj_fdfile_t *self = self_in; fsync(self->fd); +#else + //TODO +#endif return mp_const_none; } STATIC MP_DEFINE_CONST_FUN_OBJ_1(fdfile_flush_obj, fdfile_flush);