mirrors/micropython
1
0
Fork 0
mirror of https://github.com/licsber/micropython.git synced 2024-09-20 09:00:23 +08:00
Code Issues Packages Projects Releases Wiki Activity
a1c8e5737c
micropython/py/lexer.c
Damien d99b05282d Change object representation from 1 big union to individual structs. 
A big change.  Micro Python objects are allocated as individual structs
with the first element being a pointer to the type information (which
is itself an object).  This scheme follows CPython.  Much more flexible,
not necessarily slower, uses same heap memory, and can allocate objects
statically.

Also change name prefix, from py_ to mp_ (mp for Micro Python).
2013-12-21 18:17:45 +00:00

688 lines
21 KiB
C
Raw Blame History

/* lexer.c -- simple tokeniser for Python implementation
*/
#include <stdint.h>
#include <stdio.h>
#include <assert.h>
#include "misc.h"
#include "lexer.h"
#define TAB_SIZE (8)
// TODO seems that CPython allows NULL byte in the input stream
// don't know if that's intentional or not, but we don't allow it
struct _mp_lexer_t {
const char *name; // name of source
void *stream_data; // data for stream
mp_lexer_stream_next_char_t stream_next_char; // stream callback to get next char
mp_lexer_stream_close_t stream_close; // stream callback to free
unichar chr0, chr1, chr2; // current cached characters from source
uint line; // source line
uint column; // source column
int emit_dent; // non-zero when there are INDENT/DEDENT tokens to emit
int nested_bracket_level; // >0 when there are nested brackets over multiple lines
uint alloc_indent_level;
uint num_indent_level;
uint16_t *indent_level;
vstr_t vstr;
mp_token_t tok_cur;
};
bool str_strn_equal(const char *str, const char *strn, int len) {
uint i = 0;
while (i < len && *str == *strn) {
++i;
++str;
++strn;
}
return i == len && *str == 0;
}
void mp_token_show(const mp_token_t *tok) {
printf("(%s:%d:%d) kind:%d str:%p len:%d", tok->src_name, tok->src_line, tok->src_column, tok->kind, tok->str, tok->len);
if (tok->str != NULL && tok->len > 0) {
const char *i = tok->str;
const char *j = i + tok->len;
printf(" ");
while (i < j) {
unichar c = g_utf8_get_char(i);
i = g_utf8_next_char(i);
if (g_unichar_isprint(c)) {
printf("%c", c);
} else {
printf("?");
}
}
}
printf("\n");
}
void mp_token_show_error_prefix(const mp_token_t *tok) {
printf("(%s:%d:%d) ", tok->src_name, tok->src_line, tok->src_column);
}
bool mp_token_show_error(const mp_token_t *tok, const char *msg) {
printf("(%s:%d:%d) %s\n", tok->src_name, tok->src_line, tok->src_column, msg);
return false;
}
#define CUR_CHAR(lex) ((lex)->chr0)
static bool is_end(mp_lexer_t *lex) {
return lex->chr0 == MP_LEXER_CHAR_EOF;
}
static bool is_physical_newline(mp_lexer_t *lex) {
return lex->chr0 == '\n' || lex->chr0 == '\r';
}
static bool is_char(mp_lexer_t *lex, char c) {
return lex->chr0 == c;
}
static bool is_char_or(mp_lexer_t *lex, char c1, char c2) {
return lex->chr0 == c1 || lex->chr0 == c2;
}
static bool is_char_or3(mp_lexer_t *lex, char c1, char c2, char c3) {
return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3;
}
/*
static bool is_char_following(mp_lexer_t *lex, char c) {
return lex->chr1 == c;
}
*/
static bool is_char_following_or(mp_lexer_t *lex, char c1, char c2) {
return lex->chr1 == c1 || lex->chr1 == c2;
}
static bool is_char_following_following_or(mp_lexer_t *lex, char c1, char c2) {
return lex->chr2 == c1 || lex->chr2 == c2;
}
static bool is_char_and(mp_lexer_t *lex, char c1, char c2) {
return lex->chr0 == c1 && lex->chr1 == c2;
}
static bool is_whitespace(mp_lexer_t *lex) {
return g_unichar_isspace(lex->chr0);
}
static bool is_letter(mp_lexer_t *lex) {
return g_unichar_isalpha(lex->chr0);
}
static bool is_digit(mp_lexer_t *lex) {
return g_unichar_isdigit(lex->chr0);
}
static bool is_following_digit(mp_lexer_t *lex) {
return g_unichar_isdigit(lex->chr1);
}
// TODO UNICODE include unicode characters in definition of identifiers
static bool is_head_of_identifier(mp_lexer_t *lex) {
return is_letter(lex) || lex->chr0 == '_';
}
// TODO UNICODE include unicode characters in definition of identifiers
static bool is_tail_of_identifier(mp_lexer_t *lex) {
return is_head_of_identifier(lex) || is_digit(lex);
}
static void next_char(mp_lexer_t *lex) {
if (lex->chr0 == MP_LEXER_CHAR_EOF) {
return;
}
int advance = 1;
if (lex->chr0 == '\n') {
// LF is a new line
++lex->line;
lex->column = 1;
} else if (lex->chr0 == '\r') {
// CR is a new line
++lex->line;
lex->column = 1;
if (lex->chr1 == '\n') {
// CR LF is a single new line
advance = 2;
}
} else if (lex->chr0 == '\t') {
// a tab
lex->column = (((lex->column - 1 + TAB_SIZE) / TAB_SIZE) * TAB_SIZE) + 1;
} else {
// a character worth one column
++lex->column;
}
for (; advance > 0; advance--) {
lex->chr0 = lex->chr1;
lex->chr1 = lex->chr2;
lex->chr2 = lex->stream_next_char(lex->stream_data);
if (lex->chr2 == MP_LEXER_CHAR_EOF) {
// EOF
if (lex->chr1 != MP_LEXER_CHAR_EOF && lex->chr1 != '\n' && lex->chr1 != '\r') {
lex->chr2 = '\n'; // insert newline at end of file
}
}
}
}
void indent_push(mp_lexer_t *lex, uint indent) {
if (lex->num_indent_level >= lex->alloc_indent_level) {
lex->alloc_indent_level *= 2;
lex->indent_level = m_renew(uint16_t, lex->indent_level, lex->alloc_indent_level);
}
lex->indent_level[lex->num_indent_level++] = indent;
}
uint indent_top(mp_lexer_t *lex) {
return lex->indent_level[lex->num_indent_level - 1];
}
void indent_pop(mp_lexer_t *lex) {
lex->num_indent_level -= 1;
}
// some tricky operator encoding:
// <op> = begin with <op>, if this opchar matches then begin here
// e<op> = end with <op>, if this opchar matches then end
// E<op> = mandatory end with <op>, this opchar must match, then end
// c<op> = continue with <op>, if this opchar matches then continue matching
// this means if the start of two ops are the same then they are equal til the last char
static const char *tok_enc =
"()[]{},:;@~" // singles
"<e=c<e=" // < <= << <<=
">e=c>e=" // > >= >> >>=
"*e=c*e=" // * *= ** **=
"+e=" // + +=
"-e=e>" // - -= ->
"&e=" // & &=
"|e=" // | |=
"/e=c/e=" // / /= // //=
"%e=" // % %=
"^e=" // ^ ^=
"=e=" // = ==
"!E=" // !=
".c.E."; // . ...
// TODO static assert that number of tokens is less than 256 so we can safely make this table with byte sized entries
static const uint8_t tok_enc_kind[] = {
MP_TOKEN_DEL_PAREN_OPEN, MP_TOKEN_DEL_PAREN_CLOSE,
MP_TOKEN_DEL_BRACKET_OPEN, MP_TOKEN_DEL_BRACKET_CLOSE,
MP_TOKEN_DEL_BRACE_OPEN, MP_TOKEN_DEL_BRACE_CLOSE,
MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_DEL_AT, MP_TOKEN_OP_TILDE,
MP_TOKEN_OP_LESS, MP_TOKEN_OP_LESS_EQUAL, MP_TOKEN_OP_DBL_LESS, MP_TOKEN_DEL_DBL_LESS_EQUAL,
MP_TOKEN_OP_MORE, MP_TOKEN_OP_MORE_EQUAL, MP_TOKEN_OP_DBL_MORE, MP_TOKEN_DEL_DBL_MORE_EQUAL,
MP_TOKEN_OP_STAR, MP_TOKEN_DEL_STAR_EQUAL, MP_TOKEN_OP_DBL_STAR, MP_TOKEN_DEL_DBL_STAR_EQUAL,
MP_TOKEN_OP_PLUS, MP_TOKEN_DEL_PLUS_EQUAL,
MP_TOKEN_OP_MINUS, MP_TOKEN_DEL_MINUS_EQUAL, MP_TOKEN_DEL_MINUS_MORE,
MP_TOKEN_OP_AMPERSAND, MP_TOKEN_DEL_AMPERSAND_EQUAL,
MP_TOKEN_OP_PIPE, MP_TOKEN_DEL_PIPE_EQUAL,
MP_TOKEN_OP_SLASH, MP_TOKEN_DEL_SLASH_EQUAL, MP_TOKEN_OP_DBL_SLASH, MP_TOKEN_DEL_DBL_SLASH_EQUAL,
MP_TOKEN_OP_PERCENT, MP_TOKEN_DEL_PERCENT_EQUAL,
MP_TOKEN_OP_CARET, MP_TOKEN_DEL_CARET_EQUAL,
MP_TOKEN_DEL_EQUAL, MP_TOKEN_OP_DBL_EQUAL,
MP_TOKEN_OP_NOT_EQUAL,
MP_TOKEN_DEL_PERIOD, MP_TOKEN_ELLIPSES,
};
// must have the same order as enum in lexer.h
static const char *tok_kw[] = {
"False",
"None",
"True",
"and",
"as",
"assert",
"break",
"class",
"continue",
"def",
"del",
"elif",
"else",
"except",
"finally",
"for",
"from",
"global",
"if",
"import",
"in",
"is",
"lambda",
"nonlocal",
"not",
"or",
"pass",
"raise",
"return",
"try",
"while",
"with",
"yield",
NULL,
};
static void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool first_token) {
// skip white space and comments
bool had_physical_newline = false;
while (!is_end(lex)) {
if (is_physical_newline(lex)) {
had_physical_newline = true;
next_char(lex);
} else if (is_whitespace(lex)) {
next_char(lex);
} else if (is_char(lex, '#')) {
next_char(lex);
while (!is_end(lex) && !is_physical_newline(lex)) {
next_char(lex);
}
// had_physical_newline will be set on next loop
} else if (is_char(lex, '\\')) {
// backslash (outside string literals) must appear just before a physical newline
next_char(lex);
if (!is_physical_newline(lex)) {
// TODO SyntaxError
assert(0);
} else {
next_char(lex);
}
} else {
break;
}
}
// set token source information
tok->src_name = lex->name;
tok->src_line = lex->line;
tok->src_column = lex->column;
// start new token text
vstr_reset(&lex->vstr);
if (first_token && lex->line == 1 && lex->column != 1) {
// check that the first token is in the first column
// if first token is not on first line, we get a physical newline and
// this check is done as part of normal indent/dedent checking below
// (done to get equivalence with CPython)
tok->kind = MP_TOKEN_INDENT;
} else if (lex->emit_dent < 0) {
tok->kind = MP_TOKEN_DEDENT;
lex->emit_dent += 1;
} else if (lex->emit_dent > 0) {
tok->kind = MP_TOKEN_INDENT;
lex->emit_dent -= 1;
} else if (had_physical_newline && lex->nested_bracket_level == 0) {
tok->kind = MP_TOKEN_NEWLINE;
uint num_spaces = lex->column - 1;
lex->emit_dent = 0;
if (num_spaces == indent_top(lex)) {
} else if (num_spaces > indent_top(lex)) {
indent_push(lex, num_spaces);
lex->emit_dent += 1;
} else {
while (num_spaces < indent_top(lex)) {
indent_pop(lex);
lex->emit_dent -= 1;
}
if (num_spaces != indent_top(lex)) {
tok->kind = MP_TOKEN_DEDENT_MISMATCH;
}
}
} else if (is_end(lex)) {
if (indent_top(lex) > 0) {
tok->kind = MP_TOKEN_NEWLINE;
lex->emit_dent = 0;
while (indent_top(lex) > 0) {
indent_pop(lex);
lex->emit_dent -= 1;
}
} else {
tok->kind = MP_TOKEN_END;
}
} else if (is_char_or(lex, '\'', '\"')
|| (is_char_or3(lex, 'r', 'u', 'b') && is_char_following_or(lex, '\'', '\"'))
|| ((is_char_and(lex, 'r', 'b') || is_char_and(lex, 'b', 'r')) && is_char_following_following_or(lex, '\'', '\"'))) {
// a string or bytes literal
// parse type codes
bool is_raw = false;
bool is_bytes = false;
if (is_char(lex, 'u')) {
next_char(lex);
} else if (is_char(lex, 'b')) {
is_bytes = true;
next_char(lex);
if (is_char(lex, 'r')) {
is_raw = true;
next_char(lex);
}
} else if (is_char(lex, 'r')) {
is_raw = true;
next_char(lex);
if (is_char(lex, 'b')) {
is_bytes = true;
next_char(lex);
}
}
// set token kind
if (is_bytes) {
tok->kind = MP_TOKEN_BYTES;
} else {
tok->kind = MP_TOKEN_STRING;
}
// get first quoting character
char quote_char = '\'';
if (is_char(lex, '\"')) {
quote_char = '\"';
}
next_char(lex);
// work out if it's a single or triple quoted literal
int num_quotes;
if (is_char_and(lex, quote_char, quote_char)) {
// triple quotes
next_char(lex);
next_char(lex);
num_quotes = 3;
} else {
// single quotes
num_quotes = 1;
}
// parse the literal
int n_closing = 0;
while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) {
if (is_char(lex, quote_char)) {
n_closing += 1;
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
} else {
n_closing = 0;
if (!is_raw && is_char(lex, '\\')) {
next_char(lex);
unichar c = CUR_CHAR(lex);
switch (c) {
case MP_LEXER_CHAR_EOF: break; // TODO a proper error message?
case '\n': c = MP_LEXER_CHAR_EOF; break; // TODO check this works correctly (we are supposed to ignore it
case '\\': break;
case '\'': break;
case '"': break;
case 'a': c = 0x07; break;
case 'b': c = 0x08; break;
case 't': c = 0x09; break;
case 'n': c = 0x0a; break;
case 'v': c = 0x0b; break;
case 'f': c = 0x0c; break;
case 'r': c = 0x0d; break;
// TODO \ooo octal
case 'x': // TODO \xhh
case 'N': // TODO \N{name} only in strings
case 'u': // TODO \uxxxx only in strings
case 'U': // TODO \Uxxxxxxxx only in strings
default: break; // TODO error message
}
if (c != MP_LEXER_CHAR_EOF) {
vstr_add_char(&lex->vstr, c);
}
} else {
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
}
}
next_char(lex);
}
// check we got the required end quotes
if (n_closing < num_quotes) {
tok->kind = MP_TOKEN_LONELY_STRING_OPEN;
}
// cut off the end quotes from the token text
vstr_cut_tail(&lex->vstr, n_closing);
} else if (is_head_of_identifier(lex)) {
tok->kind = MP_TOKEN_NAME;
// get first char
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
next_char(lex);
// get tail chars
while (!is_end(lex) && is_tail_of_identifier(lex)) {
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
next_char(lex);
}
} else if (is_digit(lex) || (is_char(lex, '.') && is_following_digit(lex))) {
tok->kind = MP_TOKEN_NUMBER;
// get first char
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
next_char(lex);
// get tail chars
while (!is_end(lex)) {
if (is_char_or(lex, 'e', 'E')) {
vstr_add_char(&lex->vstr, 'e');
next_char(lex);
if (is_char(lex, '+') || is_char(lex, '-')) {
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
next_char(lex);
}
} else if (is_letter(lex) || is_digit(lex) || is_char_or(lex, '_', '.')) {
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
next_char(lex);
} else {
break;
}
}
} else {
// search for encoded delimiter or operator
const char *t = tok_enc;
uint tok_enc_index = 0;
for (; *t != 0 && !is_char(lex, *t); t += 1) {
if (*t == 'e' || *t == 'c') {
t += 1;
} else if (*t == 'E') {
tok_enc_index -= 1;
t += 1;
}
tok_enc_index += 1;
}
next_char(lex);
if (*t == 0) {
// didn't match any delimiter or operator characters
tok->kind = MP_TOKEN_INVALID;
} else {
// matched a delimiter or operator character
// get the maximum characters for a valid token
t += 1;
uint t_index = tok_enc_index;
for (;;) {
for (; *t == 'e'; t += 1) {
t += 1;
t_index += 1;
if (is_char(lex, *t)) {
next_char(lex);
tok_enc_index = t_index;
break;
}
}
if (*t == 'E') {
t += 1;
if (is_char(lex, *t)) {
next_char(lex);
tok_enc_index = t_index;
} else {
tok->kind = MP_TOKEN_INVALID;
}
break;
}
if (*t == 'c') {
t += 1;
t_index += 1;
if (is_char(lex, *t)) {
next_char(lex);
tok_enc_index = t_index;
t += 1;
} else {
break;
}
} else {
break;
}
}
// set token kind
tok->kind = tok_enc_kind[tok_enc_index];
// compute bracket level for implicit line joining
if (tok->kind == MP_TOKEN_DEL_PAREN_OPEN || tok->kind == MP_TOKEN_DEL_BRACKET_OPEN || tok->kind == MP_TOKEN_DEL_BRACE_OPEN) {
lex->nested_bracket_level += 1;
} else if (tok->kind == MP_TOKEN_DEL_PAREN_CLOSE || tok->kind == MP_TOKEN_DEL_BRACKET_CLOSE || tok->kind == MP_TOKEN_DEL_BRACE_CLOSE) {
lex->nested_bracket_level -= 1;
}
}
}
// point token text to vstr buffer
tok->str = vstr_str(&lex->vstr);
tok->len = vstr_len(&lex->vstr);
// check for keywords
if (tok->kind == MP_TOKEN_NAME) {
for (int i = 0; tok_kw[i] != NULL; i++) {
if (str_strn_equal(tok_kw[i], tok->str, tok->len)) {
tok->kind = MP_TOKEN_KW_FALSE + i;
break;
}
}
}
}
mp_lexer_t *mp_lexer_new(const char *src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close) {
mp_lexer_t *lex = m_new(mp_lexer_t, 1);
lex->name = src_name; // TODO do we need to strdup this?
lex->stream_data = stream_data;
lex->stream_next_char = stream_next_char;
lex->stream_close = stream_close;
lex->line = 1;
lex->column = 1;
lex->emit_dent = 0;
lex->nested_bracket_level = 0;
lex->alloc_indent_level = 16;
lex->num_indent_level = 1;
lex->indent_level = m_new(uint16_t, lex->alloc_indent_level);
lex->indent_level[0] = 0;
vstr_init(&lex->vstr);
// preload characters
lex->chr0 = stream_next_char(stream_data);
lex->chr1 = stream_next_char(stream_data);
lex->chr2 = stream_next_char(stream_data);
// if input stream is 0, 1 or 2 characters long and doesn't end in a newline, then insert a newline at the end
if (lex->chr0 == MP_LEXER_CHAR_EOF) {
lex->chr0 = '\n';
} else if (lex->chr1 == MP_LEXER_CHAR_EOF) {
if (lex->chr0 != '\n' && lex->chr0 != '\r') {
lex->chr1 = '\n';
}
} else if (lex->chr2 == MP_LEXER_CHAR_EOF) {
if (lex->chr1 != '\n' && lex->chr1 != '\r') {
lex->chr2 = '\n';
}
}
// preload first token
mp_lexer_next_token_into(lex, &lex->tok_cur, true);
return lex;
}
void mp_lexer_free(mp_lexer_t *lex) {
if (lex) {
if (lex->stream_close) {
lex->stream_close(lex->stream_data);
}
vstr_clear(&lex->vstr);
m_free(lex);
}
}
void mp_lexer_to_next(mp_lexer_t *lex) {
mp_lexer_next_token_into(lex, &lex->tok_cur, false);
}
const mp_token_t *mp_lexer_cur(const mp_lexer_t *lex) {
return &lex->tok_cur;
}
bool mp_lexer_is_kind(mp_lexer_t *lex, mp_token_kind_t kind) {
return lex->tok_cur.kind == kind;
}
/*
bool mp_lexer_is_str(mp_lexer_t *lex, const char *str) {
return mp_token_is_str(&lex->tok_cur, str);
}
bool mp_lexer_opt_kind(mp_lexer_t *lex, mp_token_kind_t kind) {
if (mp_lexer_is_kind(lex, kind)) {
mp_lexer_to_next(lex);
return true;
}
return false;
}
bool mp_lexer_opt_str(mp_lexer_t *lex, const char *str) {
if (mp_lexer_is_str(lex, str)) {
mp_lexer_to_next(lex);
return true;
}
return false;
}
*/
bool mp_lexer_show_error(mp_lexer_t *lex, const char *msg) {
return mp_token_show_error(&lex->tok_cur, msg);
}
bool mp_lexer_show_error_pythonic(mp_lexer_t *lex, const char *msg) {
printf(" File \"%s\", line %d column %d\n%s\n", lex->tok_cur.src_name, lex->tok_cur.src_line, lex->tok_cur.src_column, msg);
return false;
}
Reference in New Issue View Git Blame Copy Permalink