diff options
| author | eunhhu <[email protected]> | 2026-09-25 13:01:12 +0100 |
|---|---|---|
| committer | eunhhu <[email protected]> | 2026-09-25 13:01:12 +0100 |
| commit | 030048d39c8ce353cca8842a9ad0ab2765290e59 (patch) | |
| tree | f6ca81ab670705b9e3d505e5410fcf2a9cd9735d | |
| parent | 3265bc65ef6d2e399be6141f678f4f299a2d7491 (diff) | |
| download | aup-030048d39c8ce353cca8842a9ad0ab2765290e59.tar.gz aup-030048d39c8ce353cca8842a9ad0ab2765290e59.zip | |
Compile text, cursors and strings to module v3
aup now emits Pure module v3 and adds text states with string initializers,
cursors bound to a text, text inputs, string literals with escapes and
{interpolation} of numbers and text, cursor distances in chars and lines,
.start/.end, insert/remove, name.line(cursor), .chars/.lines/.bytes on text,
.chars/.line on cursors, and text(bytes)/bytes(text). Replies and replace
accept text as its UTF-8 bytes. Views are numbers or text. Up to 32 inputs.
examples/notes.aup is a small editor. make check replays random typing
(ASCII, Korean, emoji, combining marks, NUL, newlines), moves and deletions
against a Python reference model: 4,800 snapshots of text, cursor, mark and
views match. It also checks escapes and number text, 16 new diagnostics, a
document loaded into text and saved back, and all earlier parity; the
compiler and the Pure executor each pass under ASan/UBSan.
| -rw-r--r-- | examples/notes.aup | 35 | ||||
| -rw-r--r-- | src/compile.c | 671 | ||||
| -rw-r--r-- | tests/check.py | 130 |
3 files changed, 696 insertions, 140 deletions
diff --git a/examples/notes.aup b/examples/notes.aup new file mode 100644 index 0000000..fdc91b8 --- /dev/null +++ b/examples/notes.aup @@ -0,0 +1,35 @@ +# A small text editor. The runtime owns the text: typing inserts at the +# cursor, the cursor and the mark always sit between characters and follow +# edits, and a failed event would leave everything as it was. +state title: text = "메모" +state note: text = "" +state at: cursor in note +state mark: cursor in note + +input typed = text +input left = key ArrowLeft +input right = key ArrowRight +input up = key ArrowUp +input down = key ArrowDown +input back = key Backspace +input forward = key Delete +input home = key Home +input finish = key End +input set_mark = key KeyM + +on typed { note.insert(at, typed) } +on left.down { at = at - 1 char } +on right.down { at = at + 1 char } +on up.down { at = at - 1 line } +on down.down { at = at + 1 line } +on back.down { note.remove(at - 1 char .. at) } +on forward.down { note.remove(at .. at + 1 char) } +on home.down { at = note.start } +on finish.down { at = note.end } +on set_mark.down { mark = at } + +view header = "{title} · {note.chars}자" +view current = note.line(at) +view position = "line {at.line}, char {at.chars}" +view size = note.bytes +view lines = note.lines diff --git a/src/compile.c b/src/compile.c index e2000ee..e95d16c 100644 --- a/src/compile.c +++ b/src/compile.c @@ -7,45 +7,57 @@ #include <stdlib.h> #include <string.h> -/* Limits and opcodes of Pure module format v2. */ -enum { STATES = 32, INPUTS = 8, VIEWS = 24, HANDLERS = 64, CODE = 512, STACK = 64, NAME = 32, - DEPTH = 64, FIELDS = 32, COLLECTIONS = 1 }; +/* Limits and opcodes of Pure module format v3. */ +enum { STATES = 32, INPUTS = 32, VIEWS = 24, HANDLERS = 64, CODE = 512, STACK = 64, NAME = 32, + DEPTH = 64, FIELDS = 32, COLLECTIONS = 1, CONSTANTS = 64, CONSTANT_BYTES = 8192 }; enum { CONST, LOAD, DT, ADD, SUB, MUL, DIV, NEG, LT, LE, GT, GE, EQ, NE, AND, OR, NOT, - MIN, MAX, CLAMP, STORE, JZ, JMP, FIELD, DOC, DOC_SIZE, LEN, REPLACE, REPLY, REQUIRE }; -enum { KIND_KEY = 1, KIND_MESSAGE = 2, FIELD_U32 = 1, FIELD_BYTES = 2 }; -typedef enum { NUM = 1, BYTES = 2 } Type; + MIN, MAX, CLAMP, STORE, JZ, JMP, FIELD, DOC, DOC_SIZE, LEN, REPLACE, REPLY, REQUIRE, + TEXT_CONST, TEXT_LOAD, TEXT_INPUT, NUM_TEXT, CONCAT, CHARS, LINES, TEXT_BYTES, + CUR_LOAD, CUR_START, CUR_END, CUR_MOVE, CUR_STORE, TEXT_INSERT, TEXT_REMOVE, TEXT_LINE, + TEXT_SET, TEXT_FROM, BYTES_FROM, CUR_COUNT }; +enum { KIND_KEY = 1, KIND_MESSAGE = 2, KIND_TEXT = 3, FIELD_U32 = 1, FIELD_BYTES = 2 }; +enum { STATE_NUM = 0, STATE_TEXT = 1, STATE_CURSOR = 2, UNIT_CHAR = 0, UNIT_LINE = 1 }; +typedef enum { NUM = 1, BYTES, TEXT, CURSOR, UNIT } Type; +typedef struct { Type type; int owner; } Kind; /* owner: a cursor's text state */ -typedef enum { T_EOF, T_NL, T_NUMBER, T_NAME, T_PUNCT } TokenKind; +typedef enum { T_EOF, T_NL, T_NUMBER, T_NAME, T_PUNCT, T_STRING } TokenKind; typedef struct { TokenKind kind; int line, column; double number; char text[NAME + 1]; /* name, or punctuation of one or two characters */ + size_t start, end; /* string contents in the source */ } Token; typedef enum { E_NUMBER, E_NAME, E_DT, E_UNARY, E_BINARY, E_CALL, - E_FIELD, /* name.member */ - E_DOC, /* name[key] */ - E_SIZE, /* name[key].bytes */ - E_LEN, /* value.bytes */ + E_MEMBER, /* args[0].member */ + E_DOC, /* name[key] */ + E_SIZE, /* name[key].bytes */ + E_STRING, /* parts linked by next: E_TEXT literals and expressions */ + E_TEXT, /* literal bytes */ + E_UNIT, /* args[0] char|line */ + E_CONVERT, /* text(args[0]) or bytes(args[0]) */ + E_LINE, /* name.line(args[0]) */ } ExprKind; typedef struct Expr { ExprKind kind; int line, column, op; double number; char name[NAME], member[NAME]; - struct Expr *args[3]; + struct Expr *args[3], *next; unsigned count; + unsigned char *bytes; + size_t size; } Expr; -typedef enum { S_ASSIGN, S_IF, S_REPLACE, S_REPLY, S_REQUIRE } StmtKind; +typedef enum { S_ASSIGN, S_IF, S_REPLACE, S_REPLY, S_REQUIRE, S_INSERT, S_REMOVE } StmtKind; typedef struct Stmt { StmtKind kind; int line, column; char name[NAME]; Expr *value; /* assigned value, condition, reply, requirement or insertion */ - Expr *first, *last; /* replace range */ + Expr *first, *last; /* a range, or the insertion cursor in first */ struct Stmt *then, *otherwise, *next; } Stmt; @@ -70,8 +82,10 @@ typedef struct { void **blocks; size_t block_count, block_capacity; /* Declarations in source order. */ - char state_names[STATES][NAME]; + char state_names[STATES][NAME], state_target[STATES][NAME]; + int state_types[STATES], state_line[STATES], state_column[STATES]; double initial[STATES]; + Expr *state_text[STATES]; unsigned states; char collection_names[COLLECTIONS][NAME]; unsigned collections; @@ -83,10 +97,15 @@ typedef struct { struct { char input[NAME]; int edge, line, column; Stmt *body; } handlers[HANDLERS]; unsigned handler_count; /* Code generation. */ - int message; /* input of the handler being generated, or -1 */ + int message; /* message or text input of the handler being generated, or -1 */ struct { uint8_t op; uint32_t arg; double value; } code[CODE]; unsigned code_count; uint32_t view_code[VIEWS][2], handler_code[HANDLERS][2]; + int view_types[VIEWS]; + unsigned char constant_data[CONSTANT_BYTES]; + uint32_t constant_offset[CONSTANTS], constant_length[CONSTANTS]; + unsigned constants; + size_t constant_used; } Compiler; static const char *const reserved[] = {"state", "input", "view", "on", "if", "else", "key", "frame", @@ -111,7 +130,7 @@ static void *allocate(Compiler *c, size_t size) { c->blocks = blocks; c->block_capacity = capacity; } - void *block = calloc(1, size); + void *block = calloc(1, size ? size : 1); if (!block) fail_at(c, c->line, c->column, "out of memory"); return c->blocks[c->block_count++] = block; } @@ -140,6 +159,23 @@ static Token scan(Compiler *c) { if (c->at >= c->length) { token.kind = T_EOF; return token; } char ch = peek_char(c, 0); if (ch == '\n') { advance(c); token.kind = T_NL; return token; } + if (ch == '"') { + /* One line; escapes and {expressions} are read by the parser. */ + advance(c); + token.kind = T_STRING; + token.start = c->at; + for (;;) { + char next = peek_char(c, 0); + if (c->at >= c->length || next == '\n' || next == '\r') + fail_at(c, token.line, token.column, "a string ends on the line it starts"); + if (next == '"') break; + if (next == '\\' && c->at + 1 < c->length) advance(c); + advance(c); + } + token.end = c->at; + advance(c); + return token; + } if (digit(ch) || (ch == '.' && digit(peek_char(c, 1)))) { /* \d+(\.\d*)?|\.\d+ with an optional exponent; a '.' that starts '..' ends the number. */ char text[400]; @@ -193,7 +229,7 @@ static Token scan(Compiler *c) { } if ((unsigned char)ch >= 32 && (unsigned char)ch < 127) fail_at(c, token.line, token.column, "unexpected character '%c'", ch); - fail_at(c, token.line, token.column, "unexpected byte 0x%02x", (unsigned char)ch); + fail_at(c, token.line, token.column, "unexpected byte 0x%02x outside a string", (unsigned char)ch); } static Token take(Compiler *c) { @@ -246,6 +282,23 @@ static bool reserved_name(const char *name) { return false; } +/* Strict UTF-8, as Pure checks its constants. */ +static bool utf8_valid(const unsigned char *bytes, size_t size) { + for (size_t i = 0; i < size;) { + unsigned char b = bytes[i]; + size_t need = b < 0x80 ? 0 : b >= 0xC2 && b <= 0xDF ? 1 : b >= 0xE0 && b <= 0xEF ? 2 : + b >= 0xF0 && b <= 0xF4 ? 3 : 4; + if (need == 4 || need > size - i - 1) return false; + unsigned low = b == 0xE0 ? 0xA0 : b == 0xF0 ? 0x90 : 0x80, high = b == 0xED ? 0x9F : b == 0xF4 ? 0x8F : 0xBF; + for (size_t k = 1; k <= need; ++k) { + unsigned char next = bytes[i + k]; + if (next < (k == 1 ? low : 0x80) || next > (k == 1 ? high : 0xBF)) return false; + } + i += need + 1; + } + return true; +} + /* ---- Parser --------------------------------------------------------- */ static const struct { const char *text; int precedence, op; } binary[] = { @@ -253,14 +306,16 @@ static const struct { const char *text; int precedence, op; } binary[] = { {">", 4, GT}, {">=", 4, GE}, {"+", 5, ADD}, {"-", 5, SUB}, {"*", 6, MUL}, {"/", 6, DIV}, }; -static Expr *node(Compiler *c, ExprKind kind, const Token *at) { +static Expr *node(Compiler *c, ExprKind kind, int line, int column) { Expr *e = allocate(c, sizeof(*e)); e->kind = kind; - e->line = at->line; - e->column = at->column; + e->line = line; + e->column = column; return e; } +static Expr *expression(Compiler *c, int minimum, bool multiline); + /* name[key]: collections are indexed only by the message key. */ static void index_key(Compiler *c) { take(c); /* [ */ @@ -271,7 +326,7 @@ static void index_key(Compiler *c) { } static Expr *postfix(Compiler *c, Expr *e) { - if (is(c, "[")) { + if (e->kind == E_NAME && is(c, "[")) { index_key(c); e->kind = E_DOC; if (is(c, ".") && lookahead(c)->kind == T_NAME && !strcmp(lookahead(c)->text, "bytes")) { @@ -283,30 +338,141 @@ static Expr *postfix(Compiler *c, Expr *e) { } while (is(c, ".")) { take(c); - Token member = expect_name(c, "a field name or bytes"); - if (!strcmp(member.text, "bytes")) { - Expr *length = node(c, E_LEN, &member); - length->args[0] = e; - length->count = 1; - e = length; - } else if (e->kind == E_NAME) { - e->kind = E_FIELD; - memcpy(e->member, member.text, NAME); - } else fail_at(c, member.line, member.column, "only bytes can follow a field"); + Token member = expect_name(c, "a field or member name"); + if (is(c, "(")) { + if (e->kind != E_NAME || strcmp(member.text, "line")) + fail_at(c, member.line, member.column, "only a text state has a method here: name.line(cursor)"); + take(c); + Expr *line = node(c, E_LINE, member.line, member.column); + memcpy(line->name, e->name, NAME); + line->args[0] = expression(c, 0, true); + line->count = 1; + newlines(c); + expect(c, ")", "')'"); + e = line; + continue; + } + Expr *access = node(c, E_MEMBER, e->line, e->column); /* errors point at the expression start */ + access->args[0] = e; + access->count = 1; + memcpy(access->member, member.text, NAME); + e = access; } return e; } +/* "text {expression} more": literal parts and expressions, escapes decoded. */ +static Expr *string(Compiler *c, const Token *token) { + Expr *s = node(c, E_STRING, token->line, token->column), **tail = &s->args[0]; + unsigned char *literal = allocate(c, token->end - token->start + 1); + size_t used = 0, at = token->start; + int literal_column = token->column; + while (at <= token->end) { + char ch = at < token->end ? c->source[at] : 0; + int column = token->column + 1 + (int)(at - token->start); + if (at == token->end || ch == '{') { + if (used || (at == token->end && !s->args[0])) { + if (!utf8_valid(literal, used)) fail_at(c, token->line, literal_column, "a string must be valid UTF-8"); + Expr *part = node(c, E_TEXT, token->line, literal_column); + part->bytes = allocate(c, used); + if (used) memcpy(part->bytes, literal, used); + part->size = used; + *tail = part; + tail = &part->next; + used = 0; + } + if (at == token->end) break; + size_t close = at + 1; + while (close < token->end && c->source[close] != '}') ++close; + if (close == token->end) fail_at(c, token->line, column, "expected '}' to end the {expression}"); + /* Parse the expression with a scanner limited to its braces. */ + size_t saved_at = c->at, saved_length = c->length; + int saved_line = c->line, saved_column = c->column; + Token saved_token = c->token, saved_ahead = c->ahead; + bool saved_has_ahead = c->has_ahead; + c->at = at + 1; + c->length = close; + c->line = token->line; + c->column = column + 1; + c->has_ahead = false; + c->token = scan(c); + if (c->token.kind == T_EOF) fail_at(c, token->line, column, "empty {expression}"); + Expr *part = expression(c, 0, false); + if (c->token.kind != T_EOF) fail_at(c, c->token.line, c->token.column, "expected '}'"); + c->at = saved_at; + c->length = saved_length; + c->line = saved_line; + c->column = saved_column; + c->token = saved_token; + c->ahead = saved_ahead; + c->has_ahead = saved_has_ahead; + *tail = part; + tail = &part->next; + at = close + 1; + literal_column = token->column + 1 + (int)(at - token->start); + continue; + } + if (ch == '\\') { + char escaped = c->source[at + 1]; + const char *simple = "n\nt\tr\r\"\"\\\\{{}}"; + const char *found = strchr(simple, escaped); + if (escaped && found && (found - simple) % 2 == 0) { + literal[used++] = (unsigned char)found[1]; + at += 2; + continue; + } + if (escaped == 'u' && c->source[at + 2] == '{') { + size_t close = at + 3; + unsigned long value = 0; + while (close < token->end && c->source[close] != '}' && close - at - 3 < 6) { + char h = c->source[close]; + int digit_value = h >= '0' && h <= '9' ? h - '0' : h >= 'a' && h <= 'f' ? h - 'a' + 10 : + h >= 'A' && h <= 'F' ? h - 'A' + 10 : -1; + if (digit_value < 0) break; + value = value * 16 + (unsigned long)digit_value; + ++close; + } + if (close == at + 3 || close >= token->end || c->source[close] != '}' || + value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF)) + fail_at(c, token->line, column, "\\u{...} needs 1-6 hex digits naming a Unicode scalar"); + if (value < 0x80) literal[used++] = (unsigned char)value; + else if (value < 0x800) { + literal[used++] = (unsigned char)(0xC0 | value >> 6); + literal[used++] = (unsigned char)(0x80 | (value & 0x3F)); + } else if (value < 0x10000) { + literal[used++] = (unsigned char)(0xE0 | value >> 12); + literal[used++] = (unsigned char)(0x80 | (value >> 6 & 0x3F)); + literal[used++] = (unsigned char)(0x80 | (value & 0x3F)); + } else { + literal[used++] = (unsigned char)(0xF0 | value >> 18); + literal[used++] = (unsigned char)(0x80 | (value >> 12 & 0x3F)); + literal[used++] = (unsigned char)(0x80 | (value >> 6 & 0x3F)); + literal[used++] = (unsigned char)(0x80 | (value & 0x3F)); + } + at = close + 1; + continue; + } + fail_at(c, token->line, column, "unknown escape; use \\n \\t \\r \\\" \\\\ \\{ \\} or \\u{...}"); + } + if (ch == '}') fail_at(c, token->line, column, "a '}' in a string is written \\}"); + literal[used++] = (unsigned char)ch; + ++at; + } + return s; +} + static Expr *expression(Compiler *c, int minimum, bool multiline) { if (++c->depth > DEPTH) fail_at(c, c->token.line, c->token.column, "expression nesting exceeds 64"); if (multiline) newlines(c); Token token = take(c); Expr *left; if (token.kind == T_NUMBER) { - left = node(c, E_NUMBER, &token); + left = node(c, E_NUMBER, token.line, token.column); left->number = token.number; + } else if (token.kind == T_STRING) { + left = string(c, &token); } else if (token.kind == T_PUNCT && (!strcmp(token.text, "-") || !strcmp(token.text, "!"))) { - left = node(c, E_UNARY, &token); + left = node(c, E_UNARY, token.line, token.column); left->op = token.text[0] == '-' ? NEG : NOT; left->args[0] = expression(c, 7, multiline); left->count = 1; @@ -316,12 +482,12 @@ static Expr *expression(Compiler *c, int minimum, bool multiline) { expect(c, ")", "')'"); } else if (token.kind == T_NAME) { static const struct { const char *name; unsigned count; int op; } calls[] = - {{"min", 2, MIN}, {"max", 2, MAX}, {"clamp", 3, CLAMP}}; + {{"min", 2, MIN}, {"max", 2, MAX}, {"clamp", 3, CLAMP}, {"text", 1, TEXT_FROM}, {"bytes", 1, BYTES_FROM}}; int call = -1; - for (int i = 0; i < 3; ++i) if (!strcmp(token.text, calls[i].name)) call = i; + for (int i = 0; i < 5; ++i) if (!strcmp(token.text, calls[i].name)) call = i; if (call >= 0 && is(c, "(")) { take(c); - left = node(c, E_CALL, &token); + left = node(c, call >= 3 ? E_CONVERT : E_CALL, token.line, token.column); left->op = calls[call].op; newlines(c); while (!is(c, ")")) { @@ -334,16 +500,26 @@ static Expr *expression(Compiler *c, int minimum, bool multiline) { } expect(c, ")", "')'"); if (left->count != calls[call].count) - fail_at(c, token.line, token.column, "%s expects %u arguments", token.text, calls[call].count); + fail_at(c, token.line, token.column, "%s expects %u argument%s", token.text, calls[call].count, + calls[call].count == 1 ? "" : "s"); } else if (!strcmp(token.text, "dt")) { - left = node(c, E_DT, &token); + left = node(c, E_DT, token.line, token.column); } else { if (reserved_name(token.text)) fail_at(c, token.line, token.column, "expected a number, state, or expression"); - left = node(c, E_NAME, &token); + left = node(c, E_NAME, token.line, token.column); memcpy(left->name, token.text, NAME); left = postfix(c, left); } } else fail_at(c, token.line, token.column, "expected a number, state, or expression"); + /* 1 char, n line: a distance that moves a cursor. */ + if (word(c, "char") || word(c, "line")) { + Token unit = take(c); + Expr *distance = node(c, E_UNIT, unit.line, unit.column); + distance->op = unit.text[0] == 'c' ? UNIT_CHAR : UNIT_LINE; + distance->args[0] = left; + distance->count = 1; + left = distance; + } for (;;) { if (multiline) newlines(c); int found = -1; @@ -351,7 +527,7 @@ static Expr *expression(Compiler *c, int minimum, bool multiline) { for (int i = 0; i < 12; ++i) if (!strcmp(c->token.text, binary[i].text)) found = i; if (found < 0 || binary[found].precedence < minimum) break; Token op = take(c); - Expr *e = node(c, E_BINARY, &op); + Expr *e = node(c, E_BINARY, op.line, op.column); e->op = binary[found].op; e->args[0] = left; e->args[1] = expression(c, binary[found].precedence + 1, multiline); @@ -364,6 +540,14 @@ static Expr *expression(Compiler *c, int minimum, bool multiline) { static Stmt *block(Compiler *c, unsigned depth); +static void range(Compiler *c, Stmt *s) { + s->first = expression(c, 0, true); + newlines(c); + expect(c, "..", "'..'"); + s->last = expression(c, 0, true); + newlines(c); +} + static Stmt *statement(Compiler *c, unsigned depth) { Token token = expect_name(c, "a statement or '}'"); Stmt *s = allocate(c, sizeof(*s)); @@ -400,17 +584,31 @@ static Stmt *statement(Compiler *c, unsigned depth) { if (strcmp(method.text, "replace")) fail_at(c, method.line, method.column, "a collection entry supports replace(first .. last, value)"); expect(c, "(", "'('"); s->kind = S_REPLACE; - s->first = expression(c, 0, true); - newlines(c); - expect(c, "..", "'..'"); - s->last = expression(c, 0, true); - newlines(c); + range(c, s); expect(c, ",", "','"); s->value = expression(c, 0, true); newlines(c); expect(c, ")", "')'"); return s; } + if (is(c, ".")) { + take(c); + Token method = expect_name(c, "insert or remove"); + expect(c, "(", "'('"); + if (!strcmp(method.text, "insert")) { + s->kind = S_INSERT; + s->first = expression(c, 0, true); + newlines(c); + expect(c, ",", "','"); + s->value = expression(c, 0, true); + newlines(c); + } else if (!strcmp(method.text, "remove")) { + s->kind = S_REMOVE; + range(c, s); + } else fail_at(c, method.line, method.column, "a text supports insert(cursor, text) and remove(first .. last)"); + expect(c, ")", "')'"); + return s; + } s->kind = S_ASSIGN; expect(c, "=", "'='"); s->value = expression(c, 0, false); @@ -438,11 +636,19 @@ static int input_index(const Compiler *c, const char *name) { return -1; } +static int state_index(const Compiler *c, const char *name) { + for (unsigned i = 0; i < c->states; ++i) if (!strcmp(c->state_names[i], name)) return (int)i; + return -1; +} + static const char *declared_kind(const Compiler *c, const char *name) { - for (unsigned i = 0; i < c->states; ++i) if (!strcmp(c->state_names[i], name)) return "state"; + static const char *const states[] = {"state", "text state", "cursor"}; + int state = state_index(c, name); + if (state >= 0) return states[c->state_types[state]]; for (unsigned i = 0; i < c->collections; ++i) if (!strcmp(c->collection_names[i], name)) return "collection"; int input = input_index(c, name); - if (input >= 0) return c->inputs[input].kind == KIND_KEY ? "key input" : "message"; + if (input >= 0) return c->inputs[input].kind == KIND_KEY ? "key input" : + c->inputs[input].kind == KIND_TEXT ? "text input" : "message"; for (unsigned i = 0; i < c->views; ++i) if (!strcmp(c->view_names[i], name)) return "view"; return NULL; } @@ -454,7 +660,8 @@ static void declare(Compiler *c, const Token *name) { } static bool key_name(const char *key) { - static const char *const named[] = {"Space", "Enter", "ArrowLeft", "ArrowRight", "ArrowUp", "ArrowDown"}; + static const char *const named[] = {"Space", "Enter", "ArrowLeft", "ArrowRight", "ArrowUp", "ArrowDown", + "Backspace", "Delete", "Home", "End", "Tab", "Escape", "PageUp", "PageDown"}; for (size_t i = 0; i < sizeof(named) / sizeof(*named); ++i) if (!strcmp(key, named[i])) return true; if (!strncmp(key, "Key", 3)) return key[3] >= 'A' && key[3] <= 'Z' && !key[4]; if (!strncmp(key, "Digit", 5)) return key[5] >= '0' && key[5] <= '9' && !key[6]; @@ -486,31 +693,63 @@ static void message_fields(Compiler *c, Input *in) { take(c); } +static void state_declaration(Compiler *c) { + Token name = expect_name(c, "state name"); + declare(c, &name); + if (is(c, ":") && lookahead(c)->kind == T_NAME && !strcmp(lookahead(c)->text, "bytes")) { + take(c); + take(c); + expect_word(c, "by"); + expect_word(c, "key"); + if (c->collections == COLLECTIONS) fail_at(c, name.line, name.column, "at most one keyed collection is allowed"); + memcpy(c->collection_names[c->collections++], name.text, NAME); + return; + } + if (c->states == STATES) fail_at(c, name.line, name.column, "at most 32 states are allowed"); + unsigned index = c->states; + memcpy(c->state_names[index], name.text, NAME); + c->state_line[index] = name.line; + c->state_column[index] = name.column; + if (is(c, ":")) { + take(c); + Token type = expect_name(c, "text or cursor"); + if (!strcmp(type.text, "text")) { + c->state_types[index] = STATE_TEXT; + if (is(c, "=")) { + take(c); + if (c->token.kind != T_STRING) fail_at(c, c->token.line, c->token.column, "a text state starts from a string"); + Token literal = take(c); + Expr *value = string(c, &literal); + if (value->args[0]->kind != E_TEXT || value->args[0]->next) + fail_at(c, literal.line, literal.column, "a text state starts from a plain string without {expressions}"); + c->state_text[index] = value->args[0]; + } + } else if (!strcmp(type.text, "cursor")) { + c->state_types[index] = STATE_CURSOR; + expect_word(c, "in"); + Token target = expect_name(c, "a text state"); + memcpy(c->state_target[index], target.text, NAME); + } else fail_at(c, type.line, type.column, "state types are numbers, text, cursor in a text, and bytes by key"); + ++c->states; + return; + } + expect(c, "=", "'='"); + double sign = 1; + if (is(c, "-") || is(c, "+")) sign = take(c).text[0] == '-' ? -1 : 1; + if (c->token.kind != T_NUMBER) fail_at(c, c->token.line, c->token.column, "expected a numeric state initializer"); + c->state_types[index] = STATE_NUM; + c->initial[index] = sign * take(c).number; + ++c->states; +} + static void declaration(Compiler *c) { Token kind = expect_name(c, "state, input, view, or on"); if (!strcmp(kind.text, "state")) { - Token name = expect_name(c, "state name"); - declare(c, &name); - if (is(c, ":")) { - take(c); - expect_word(c, "bytes"); - expect_word(c, "by"); - expect_word(c, "key"); - if (c->collections == COLLECTIONS) fail_at(c, name.line, name.column, "at most one keyed collection is allowed"); - memcpy(c->collection_names[c->collections++], name.text, NAME); - } else { - if (c->states == STATES) fail_at(c, name.line, name.column, "at most 32 states are allowed"); - expect(c, "=", "'='"); - double sign = 1; - if (is(c, "-") || is(c, "+")) sign = take(c).text[0] == '-' ? -1 : 1; - if (c->token.kind != T_NUMBER) fail_at(c, c->token.line, c->token.column, "expected a numeric state initializer"); - memcpy(c->state_names[c->states], name.text, NAME); - c->initial[c->states++] = sign * take(c).number; - } + state_declaration(c); } else if (!strcmp(kind.text, "input")) { Token name = expect_name(c, "input name"); declare(c, &name); - if (c->input_count == INPUTS) fail_at(c, name.line, name.column, "at most 8 inputs are allowed"); + if (c->input_count == INPUTS) fail_at(c, name.line, name.column, "at most 32 inputs are allowed"); expect(c, "=", "'='"); Input *in = &c->inputs[c->input_count]; memset(in, 0, sizeof(*in)); @@ -521,8 +760,12 @@ static void declaration(Compiler *c) { if (word(c, "by")) { take(c); expect_word(c, "key"); in->keyed = true; } ++c->input_count; message_fields(c, in); + } else if (word(c, "text")) { + take(c); + in->kind = KIND_TEXT; + ++c->input_count; } else { - if (!word(c, "key")) fail_at(c, c->token.line, c->token.column, "expected 'key' or 'message'"); + if (!word(c, "key")) fail_at(c, c->token.line, c->token.column, "expected 'key', 'text' or 'message'"); take(c); Token key = expect_name(c, "key name"); if (!key_name(key.text)) fail_at(c, key.line, key.column, "unknown key '%s'", key.text); @@ -572,12 +815,23 @@ static void emit(Compiler *c, int line, int column, uint8_t op, uint32_t arg, do c->code[c->code_count++].value = value; } -static int state_index(const Compiler *c, const char *name) { - for (unsigned i = 0; i < c->states; ++i) if (!strcmp(c->state_names[i], name)) return (int)i; - return -1; +static const char *type_name(Type type) { + static const char *const names[] = {"", "a number", "bytes", "text", "a cursor", "a distance"}; + return names[type]; } -static const char *type_name(Type type) { return type == NUM ? "a number" : "bytes"; } +static uint32_t constant(Compiler *c, const Expr *e) { + for (unsigned i = 0; i < c->constants; ++i) + if (c->constant_length[i] == e->size && (!e->size || !memcmp(c->constant_data + c->constant_offset[i], e->bytes, e->size))) + return i; + if (c->constants == CONSTANTS) fail_at(c, e->line, e->column, "at most 64 distinct strings are allowed"); + if (e->size > CONSTANT_BYTES - c->constant_used) fail_at(c, e->line, e->column, "strings exceed 8192 bytes"); + if (e->size) memcpy(c->constant_data + c->constant_used, e->bytes, e->size); + c->constant_offset[c->constants] = (uint32_t)c->constant_used; + c->constant_length[c->constants] = (uint32_t)e->size; + c->constant_used += e->size; + return c->constants++; +} /* The keyed collection used by name[key] in the current handler. */ static uint32_t collection(Compiler *c, const char *name, int line, int column) { @@ -588,73 +842,165 @@ static uint32_t collection(Compiler *c, const char *name, int line, int column) if (kind) fail_at(c, line, column, "'%s' is a %s, not a collection", name, kind); fail_at(c, line, column, "unknown collection '%s'", name); } - if (c->message < 0 || !c->inputs[c->message].keyed) + if (c->message < 0 || c->inputs[c->message].kind != KIND_MESSAGE || !c->inputs[c->message].keyed) fail_at(c, line, column, "%s[key] needs the handler of a message by key", name); return (uint32_t)index; } -/* Emit an expression; height is the stack height before it. Returns its type. */ -static Type value(Compiler *c, const Expr *e, bool handler, int height) { +static int text_state(Compiler *c, const char *name, int line, int column) { + int index = state_index(c, name); + if (index < 0 || c->state_types[index] != STATE_TEXT) { + const char *kind = declared_kind(c, name); + if (kind) fail_at(c, line, column, "'%s' is a %s, not a text state", name, kind); + fail_at(c, line, column, "unknown text state '%s'", name); + } + return index; +} + +static int cursor_owner(const Compiler *c, int cursor) { return state_index(c, c->state_target[cursor]); } + +static Kind value(Compiler *c, const Expr *e, bool handler, int height); + +static void expect_kind(Compiler *c, const Expr *e, Kind actual, Type type) { + if (actual.type != type) fail_at(c, e->line, e->column, "expected %s, not %s", type_name(type), type_name(actual.type)); +} + +static Kind number(Compiler *c, const Expr *e, bool handler, int height) { + Kind kind = value(c, e, handler, height); + expect_kind(c, e, kind, NUM); + return kind; +} + +/* A cursor of the named text state. */ +static void cursor_of(Compiler *c, const Expr *e, bool handler, int height, int text, const char *name) { + Kind kind = value(c, e, handler, height); + expect_kind(c, e, kind, CURSOR); + if (kind.owner != text) + fail_at(c, e->line, e->column, "a cursor of '%s' cannot point into '%s'", c->state_names[kind.owner], name); +} + +/* Emit an expression; height is the stack height before it. */ +static Kind value(Compiler *c, const Expr *e, bool handler, int height) { if (height + 1 > STACK) fail_at(c, e->line, e->column, "expression needs more than 64 stack values"); switch (e->kind) { - case E_NUMBER: emit(c, e->line, e->column, CONST, 0, e->number); return NUM; + case E_NUMBER: emit(c, e->line, e->column, CONST, 0, e->number); return (Kind){NUM, -1}; case E_DT: if (!handler) fail_at(c, e->line, e->column, "dt is only available in handlers"); emit(c, e->line, e->column, DT, 0, 0); - return NUM; + return (Kind){NUM, -1}; + case E_TEXT: emit(c, e->line, e->column, TEXT_CONST, constant(c, e), 0); return (Kind){TEXT, -1}; + case E_STRING: { + int pieces = 0; + for (const Expr *part = e->args[0]; part; part = part->next) { + Kind kind = value(c, part, handler, height + (pieces > 0)); + if (kind.type == NUM) emit(c, part->line, part->column, NUM_TEXT, 0, 0); + else if (kind.type != TEXT) + fail_at(c, part->line, part->column, "a string shows numbers and text, not %s", type_name(kind.type)); + if (pieces++) emit(c, part->line, part->column, CONCAT, 0, 0); + } + return (Kind){TEXT, -1}; + } case E_NAME: { int index = state_index(c, e->name); - if (index < 0) { - const char *kind = declared_kind(c, e->name); - if (kind) fail_at(c, e->line, e->column, "'%s' is a %s, not a state", e->name, kind); - fail_at(c, e->line, e->column, "unknown state '%s'", e->name); + if (index >= 0) { + int type = c->state_types[index]; + emit(c, e->line, e->column, type == STATE_NUM ? LOAD : type == STATE_TEXT ? TEXT_LOAD : CUR_LOAD, (uint32_t)index, 0); + return type == STATE_NUM ? (Kind){NUM, -1} : type == STATE_TEXT ? (Kind){TEXT, -1} : + (Kind){CURSOR, cursor_owner(c, index)}; } - emit(c, e->line, e->column, LOAD, (uint32_t)index, 0); - return NUM; - } - case E_FIELD: { int input = input_index(c, e->name); - if (input < 0 || c->inputs[input].kind != KIND_MESSAGE) { - const char *kind = declared_kind(c, e->name); - if (kind) fail_at(c, e->line, e->column, "'%s' is a %s and has no fields", e->name, kind); - fail_at(c, e->line, e->column, "unknown message '%s'", e->name); + if (input >= 0 && c->inputs[input].kind == KIND_TEXT) { + if (input != c->message) fail_at(c, e->line, e->column, "'%s' is only available in its handler", e->name); + emit(c, e->line, e->column, TEXT_INPUT, 0, 0); + return (Kind){TEXT, -1}; } - if (input != c->message) - fail_at(c, e->line, e->column, "fields of '%s' are only available in its handler", e->name); - const Input *in = &c->inputs[input]; - for (unsigned i = 0; i < in->fields; ++i) - if (!strcmp(in->field_names[i], e->member)) { - emit(c, e->line, e->column, FIELD, i, 0); - return in->field_types[i] == FIELD_U32 ? NUM : BYTES; + const char *kind = declared_kind(c, e->name); + if (kind) fail_at(c, e->line, e->column, "'%s' is a %s, not a state", e->name, kind); + fail_at(c, e->line, e->column, "unknown state '%s'", e->name); + } + case E_MEMBER: { + const Expr *base = e->args[0]; + if (base->kind == E_NAME) { + int input = input_index(c, base->name); + if (input >= 0 && c->inputs[input].kind == KIND_MESSAGE) { + if (input != c->message) + fail_at(c, e->line, e->column, "fields of '%s' are only available in its handler", base->name); + const Input *in = &c->inputs[input]; + for (unsigned i = 0; i < in->fields; ++i) + if (!strcmp(in->field_names[i], e->member)) { + emit(c, e->line, e->column, FIELD, i, 0); + return (Kind){in->field_types[i] == FIELD_U32 ? NUM : BYTES, -1}; + } + fail_at(c, e->line, e->column, "message '%s' has no field '%s'", base->name, e->member); } - fail_at(c, e->line, e->column, "message '%s' has no field '%s'", e->name, e->member); + if (input >= 0 && c->inputs[input].kind == KIND_KEY) + fail_at(c, base->line, base->column, "'%s' is a key input and has no fields", base->name); + int state = state_index(c, base->name); + if (state >= 0 && c->state_types[state] == STATE_TEXT && + (!strcmp(e->member, "start") || !strcmp(e->member, "end"))) { + emit(c, e->line, e->column, e->member[0] == 's' ? CUR_START : CUR_END, (uint32_t)state, 0); + return (Kind){CURSOR, state}; + } + } + Kind inner = value(c, base, handler, height); + const char *m = e->member; + if (inner.type == TEXT && (!strcmp(m, "chars") || !strcmp(m, "lines") || !strcmp(m, "bytes"))) { + emit(c, e->line, e->column, m[0] == 'c' ? CHARS : m[0] == 'l' ? LINES : TEXT_BYTES, 0, 0); + return (Kind){NUM, -1}; + } + if (inner.type == BYTES && !strcmp(m, "bytes")) { emit(c, e->line, e->column, LEN, 0, 0); return (Kind){NUM, -1}; } + if (inner.type == CURSOR && (!strcmp(m, "chars") || !strcmp(m, "line"))) { + emit(c, e->line, e->column, CUR_COUNT, m[0] == 'c' ? UNIT_CHAR : UNIT_LINE, 0); + return (Kind){NUM, -1}; + } + fail_at(c, e->line, e->column, "%s has no '%s'", type_name(inner.type), m); + } + case E_LINE: { + int text = text_state(c, e->name, e->line, e->column); + cursor_of(c, e->args[0], handler, height, text, e->name); + emit(c, e->line, e->column, TEXT_LINE, (uint32_t)text, 0); + return (Kind){TEXT, -1}; } case E_DOC: emit(c, e->line, e->column, DOC, collection(c, e->name, e->line, e->column), 0); - return BYTES; + return (Kind){BYTES, -1}; case E_SIZE: emit(c, e->line, e->column, DOC_SIZE, collection(c, e->name, e->line, e->column), 0); - return NUM; - case E_LEN: { - Type inner = value(c, e->args[0], handler, height); - if (inner != BYTES) fail_at(c, e->line, e->column, ".bytes needs bytes, not %s", type_name(inner)); - emit(c, e->line, e->column, LEN, 0, 0); - return NUM; + return (Kind){NUM, -1}; + case E_CONVERT: { + Kind inner = value(c, e->args[0], handler, height); + Type from = e->op == TEXT_FROM ? BYTES : TEXT; + if (inner.type != from) + fail_at(c, e->line, e->column, "%s(...) converts %s, not %s", e->op == TEXT_FROM ? "text" : "bytes", + type_name(from), type_name(inner.type)); + emit(c, e->line, e->column, (uint8_t)e->op, 0, 0); + return (Kind){e->op == TEXT_FROM ? TEXT : BYTES, -1}; } - default: - for (unsigned i = 0; i < e->count; ++i) { - Type operand = value(c, e->args[i], handler, height + (int)i); - if (operand != NUM) - fail_at(c, e->args[i]->line, e->args[i]->column, "expected a number, not %s", type_name(operand)); + case E_UNIT: + fail_at(c, e->line, e->column, "a distance moves a cursor, as in at + 1 char"); + case E_BINARY: + if (e->args[1]->kind == E_UNIT && (e->op == ADD || e->op == SUB)) { + Kind cursor = value(c, e->args[0], handler, height); + if (cursor.type != CURSOR) + fail_at(c, e->line, e->column, "only a cursor moves by a distance, not %s", type_name(cursor.type)); + number(c, e->args[1]->args[0], handler, height + 1); + if (e->op == SUB) emit(c, e->line, e->column, NEG, 0, 0); + emit(c, e->line, e->column, CUR_MOVE, (uint32_t)e->args[1]->op, 0); + return cursor; } + /* fall through */ + default: + for (unsigned i = 0; i < e->count; ++i) number(c, e->args[i], handler, height + (int)i); emit(c, e->line, e->column, (uint8_t)e->op, 0, 0); - return NUM; + return (Kind){NUM, -1}; } } -static void expect_type(Compiler *c, const Expr *e, Type type, bool handler) { - Type actual = value(c, e, handler, 0 /* statement operands start at the top */); - if (actual != type) fail_at(c, e->line, e->column, "expected %s, not %s", type_name(type), type_name(actual)); +/* Bytes, or text sent as its UTF-8 bytes. */ +static void bytes_value(Compiler *c, const Expr *e, int height, const char *what) { + Kind kind = value(c, e, true, height); + if (kind.type == TEXT) emit(c, e->line, e->column, BYTES_FROM, 0, 0); + else if (kind.type != BYTES) fail_at(c, e->line, e->column, "%s takes bytes or text, not %s", what, type_name(kind.type)); } static void statements(Compiler *c, const Stmt *s) { @@ -667,31 +1013,53 @@ static void statements(Compiler *c, const Stmt *s) { if (kind) fail_at(c, s->line, s->column, "cannot assign to %s '%s'", kind, s->name); fail_at(c, s->line, s->column, "unknown state '%s'", s->name); } - expect_type(c, s->value, NUM, true); - emit(c, s->line, s->column, STORE, (uint32_t)index, 0); + if (c->state_types[index] == STATE_NUM) { + number(c, s->value, true, 0); + emit(c, s->line, s->column, STORE, (uint32_t)index, 0); + } else if (c->state_types[index] == STATE_TEXT) { + expect_kind(c, s->value, value(c, s->value, true, 0), TEXT); + emit(c, s->line, s->column, TEXT_SET, (uint32_t)index, 0); + } else { + int owner = cursor_owner(c, index); + cursor_of(c, s->value, true, 0, owner, c->state_names[owner]); + emit(c, s->line, s->column, CUR_STORE, (uint32_t)index, 0); + } + break; + } + case S_INSERT: { + int text = text_state(c, s->name, s->line, s->column); + cursor_of(c, s->first, true, 0, text, s->name); + expect_kind(c, s->value, value(c, s->value, true, 1), TEXT); + emit(c, s->line, s->column, TEXT_INSERT, (uint32_t)text, 0); + break; + } + case S_REMOVE: { + int text = text_state(c, s->name, s->line, s->column); + cursor_of(c, s->first, true, 0, text, s->name); + cursor_of(c, s->last, true, 1, text, s->name); + emit(c, s->line, s->column, TEXT_REMOVE, (uint32_t)text, 0); break; } case S_REPLACE: { uint32_t target = collection(c, s->name, s->line, s->column); - expect_type(c, s->first, NUM, true); - Type last = value(c, s->last, true, 1); - if (last != NUM) fail_at(c, s->last->line, s->last->column, "expected a number, not bytes"); - Type insert = value(c, s->value, true, 2); - if (insert != BYTES) fail_at(c, s->value->line, s->value->column, "replace inserts bytes, not a number"); + number(c, s->first, true, 0); + number(c, s->last, true, 1); + bytes_value(c, s->value, 2, "replace"); emit(c, s->line, s->column, REPLACE, target, 0); break; } case S_REPLY: - if (c->message < 0) fail_at(c, s->line, s->column, "reply is only available in a message handler"); - expect_type(c, s->value, BYTES, true); + if (c->message < 0 || c->inputs[c->message].kind != KIND_MESSAGE) + fail_at(c, s->line, s->column, "reply is only available in a message handler"); + bytes_value(c, s->value, 0, "reply"); emit(c, s->line, s->column, REPLY, 0, 0); break; case S_REQUIRE: - expect_type(c, s->value, NUM, true); + number(c, s->value, true, 0); emit(c, s->line, s->column, REQUIRE, 0, 0); break; case S_IF: { - expect_type(c, s->value, NUM, true); + number(c, s->value, true, 0); unsigned jump = c->code_count; emit(c, s->line, s->column, JZ, 0, 0); statements(c, s->then); @@ -739,8 +1107,20 @@ static uint32_t intern(unsigned char *strings, size_t *used, const char *text) { } static void generate(Compiler *c, unsigned char **module, size_t *size) { - uint32_t events[HANDLERS]; + uint32_t events[HANDLERS], state_ref[STATES] = {0}; int handler_input[HANDLERS]; + for (unsigned i = 0; i < c->states; ++i) { + if (c->state_types[i] == STATE_TEXT) { + Expr empty = {.kind = E_TEXT, .line = c->state_line[i], .column = c->state_column[i]}; + state_ref[i] = constant(c, c->state_text[i] ? c->state_text[i] : &empty); + } else if (c->state_types[i] == STATE_CURSOR) { + int target = state_index(c, c->state_target[i]); + if (target < 0 || c->state_types[target] != STATE_TEXT) + fail_at(c, c->state_line[i], c->state_column[i], "a cursor moves in a text state; '%s' is not one", + c->state_target[i]); + state_ref[i] = (uint32_t)target; + } + } for (unsigned i = 0; i < c->handler_count; ++i) { const char *name = c->handlers[i].input; int line = c->handlers[i].line, column = c->handlers[i].column; @@ -753,8 +1133,9 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) { if (kind) fail_at(c, line, column, "'%s' is a %s, not an input", name, kind); fail_at(c, line, column, "unknown input '%s'", name); } - if (c->inputs[input].kind == KIND_MESSAGE && c->handlers[i].edge >= 0) - fail_at(c, line, column, "'%s' is a message; write on %s { }", name, name); + const char *what = c->inputs[input].kind == KIND_MESSAGE ? "message" : "text input"; + if (c->inputs[input].kind != KIND_KEY && c->handlers[i].edge >= 0) + fail_at(c, line, column, "'%s' is a %s; write on %s { }", name, what, name); if (c->inputs[input].kind == KIND_KEY && c->handlers[i].edge < 0) fail_at(c, line, column, "'%s' is a key input; write on %s.down or on %s.up", name, name, name); handler_input[i] = input; @@ -764,13 +1145,15 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) { c->message = -1; for (unsigned i = 0; i < c->views; ++i) { c->view_code[i][0] = c->code_count; - Type type = value(c, c->view_value[i], false, 0); - if (type != NUM) fail_at(c, c->view_value[i]->line, c->view_value[i]->column, "a view is a number"); + Kind kind = value(c, c->view_value[i], false, 0); + if (kind.type != NUM && kind.type != TEXT) + fail_at(c, c->view_value[i]->line, c->view_value[i]->column, "a view is a number or text, not %s", type_name(kind.type)); + c->view_types[i] = kind.type == TEXT; c->view_code[i][1] = c->code_count - c->view_code[i][0]; } for (unsigned i = 0; i < c->handler_count; ++i) { int input = handler_input[i]; - c->message = input >= 0 && c->inputs[input].kind == KIND_MESSAGE ? input : -1; + c->message = input >= 0 && c->inputs[input].kind != KIND_KEY ? input : -1; c->handler_code[i][0] = c->code_count; statements(c, c->handlers[i].body); c->handler_code[i][1] = c->code_count - c->handler_code[i][0]; @@ -789,10 +1172,10 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) { field_name[fields++] = intern(strings, &used, c->inputs[i].field_names[f]); } for (unsigned i = 0; i < c->views; ++i) view_name[i] = intern(strings, &used, c->view_names[i]); - *size = 32 + 16u * c->states + 12u * c->collections + 16u * c->input_count + 8u * fields + - 12u * (c->views + c->handler_count) + 16u * c->code_count + used; + *size = 40 + 16u * c->states + 12u * c->collections + 16u * c->input_count + 8u * fields + 8u * c->constants + + 16u * c->views + 12u * c->handler_count + 16u * c->code_count + used + c->constant_used; unsigned char *p = *module = allocate(c, *size); - memcpy(p, "PUREMOD2", 8); + memcpy(p, "PUREMOD3", 8); put16(p + 8, c->states); put16(p + 10, c->input_count); put16(p + 12, c->views); @@ -801,8 +1184,15 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) { put32(p + 20, (uint32_t)used); put16(p + 24, fields); put16(p + 26, c->collections); - p += 32; - for (unsigned i = 0; i < c->states; ++i) { p = put32(p, state_name[i]); p = put32(p, 0); p = put_double(p, c->initial[i]); } + put16(p + 28, c->constants); + put32(p + 32, (uint32_t)c->constant_used); + p += 40; + for (unsigned i = 0; i < c->states; ++i) { + p = put32(p, state_name[i]); + p = put32(p, (uint32_t)c->state_types[i]); + if (c->state_types[i] == STATE_NUM) p = put_double(p, c->initial[i]); + else { p = put32(p, state_ref[i]); p = put32(p, 0); } + } for (unsigned i = 0; i < c->collections; ++i) { p = put32(p, collection_name[i]); p = put32(p, 1); p = put32(p, 1); } unsigned first = 0; for (unsigned i = 0; i < c->input_count; ++i) { @@ -822,7 +1212,13 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) { p = put32(p, field_name[fields++]); p = put32(p, (uint32_t)c->inputs[i].field_types[f]); } - for (unsigned i = 0; i < c->views; ++i) { p = put32(p, view_name[i]); p = put32(p, c->view_code[i][0]); p = put32(p, c->view_code[i][1]); } + for (unsigned i = 0; i < c->constants; ++i) { p = put32(p, c->constant_offset[i]); p = put32(p, c->constant_length[i]); } + for (unsigned i = 0; i < c->views; ++i) { + p = put32(p, view_name[i]); + p = put32(p, (uint32_t)c->view_types[i]); + p = put32(p, c->view_code[i][0]); + p = put32(p, c->view_code[i][1]); + } for (unsigned i = 0; i < c->handler_count; ++i) { p = put32(p, events[i]); p = put32(p, c->handler_code[i][0]); @@ -834,6 +1230,7 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) { p = put_double(p, c->code[i].value); } memcpy(p, strings, used); + if (c->constant_used) memcpy(p + used, c->constant_data, c->constant_used); } bool aup_compile(const char *source, size_t length, unsigned char **module, size_t *size, diff --git a/tests/check.py b/tests/check.py index 83757ed..5ed90ff 100644 --- a/tests/check.py +++ b/tests/check.py @@ -10,6 +10,10 @@ counts and event failures as the workbench Zig engine (via its Node CLI). Workbench JSON prints -0 as 0, so signed zeros compare numerically there; Pure's unit tests check signed zeros directly. Other values compare exactly. +6. Text model: examples/notes.aup, run by pure-run, keeps its text, cursor and + mark exactly as a reference Python model under random typing (ASCII, + Korean, emoji, combining marks, NUL, newlines), moves and deletions; string + escapes and number text follow their stated rules. 5. Editor parity: the document editor written in aup, run by pure-edit-module, answers random show/edit/apply operations exactly as pure-edit running the original editor.pure (stdout, stderr, exit code), on the same tree store @@ -31,7 +35,7 @@ DIAGNOSTICS = [ ('state a = b\n', '1:11', 'expected a numeric state initializer'), ('view v = dt\n', '1:10', 'dt is only available in handlers'), ('input k = key Space\ninput j = key Space\n', '2:15', "key Space is already used by input 'k'"), - ('input k = key Tab\n', '1:15', "unknown key 'Tab'"), + ('input k = key F13\n', '1:15', "unknown key 'F13'"), ('on missing.down {\n}\n', '1:4', "unknown input 'missing'"), ('state a = 0\non a.down {\n}\n', '2:4', "'a' is a state, not an input"), ('state a = 0\non frame {\n b = 1\n}\n', '3:3', "unknown state 'b'"), @@ -49,7 +53,7 @@ DIAGNOSTICS = [ ('state s = 0\n' * 1 + ''.join(f'state s{i} = 0\n' for i in range(32)), '33:7', 'at most 32 states are allowed'), # v0b: messages, keyed documents and types ('state d: bytes by key\ninput m = message\non m {\n reply d[key]\n}\n', '4:9', 'd[key] needs the handler of a message by key'), - ('input m = message by key { a: u32 }\non m {\n reply m.a\n}\n', '3:9', 'expected bytes, not a number'), + ('input m = message by key { a: u32 }\non m {\n reply m.a\n}\n', '3:9', 'reply takes bytes or text, not a number'), ('input m = message by key {\n b: bytes\n a: u32\n}\n', '3:3', 'a bytes field must be the last field'), ('input m = message by key\non m.down {\n}\n', '2:4', "'m' is a message; write on m { }"), ('input k = key Space\non k {\n}\n', '2:4', "'k' is a key input; write on k.down or on k.up"), @@ -58,11 +62,30 @@ DIAGNOSTICS = [ ('input m = message by key { a: u32 }\nview v = m.a\n', '2:10', "fields of 'm' are only available in its handler"), ('state x = 0\ninput m = message by key { a: bytes }\non m {\n x = m.a\n}\n', '4:7', 'expected a number, not bytes'), ('input m = message by key { a: u32 }\non m {\n reply m.z\n}\n', '3:9', "message 'm' has no field 'z'"), - ('state x = 0\non frame {\n require x.bytes\n}\n', '3:13', '.bytes needs bytes, not a number'), + ('state x = 0\non frame {\n require x.bytes\n}\n', '3:11', "a number has no 'bytes'"), ('state d: bytes by key\nstate e: bytes by key\n', '2:7', 'at most one keyed collection is allowed'), ('state d: bytes by key\ninput m = message by key { a: bytes }\non m {\n d[key].replace(0 .. m.a, m.a)\n}\n', '4:23', 'expected a number, not bytes'), ('input m = message by key { a: u32 }\non frame {\n reply m.a\n}\n', '3:3', 'reply is only available in a message handler'), + # text and cursors + ('state at: cursor in missing\n', '1:7', "a cursor moves in a text state; 'missing' is not one"), + ('state a: text\nstate b: text\nstate at: cursor in a\ninput k = key KeyK\non k.down {\n b.insert(at, "x")\n}\n', + '6:12', "a cursor of 'a' cannot point into 'b'"), + ('state a: text\nstate at: cursor in a\ninput k = key KeyK\non k.down {\n at = 3\n}\n', '5:8', + 'expected a cursor, not a number'), + ('state a: text\nstate at: cursor in a\nview v = at\n', '3:10', 'a view is a number or text, not a cursor'), + ('state a: text\nstate at: cursor in a\nview v = "{at}"\n', '3:12', 'a string shows numbers and text, not a cursor'), + ('state x = 1\nstate t: text = "{x}"\n', '2:17', 'a text state starts from a plain string'), + ('view v = "a\\qb"\n', '1:12', 'unknown escape'), + ('view v = "\\u{D800}"\n', '1:11', 'Unicode scalar'), + ('state x = 1\nview v = "{x"\n', '2:11', "expected '}' to end the {expression}"), + ('state x = 1\nview v = x + 1 char\n', '2:12', 'only a cursor moves by a distance, not a number'), + ('view v = 1 char\n', '1:12', 'a distance moves a cursor'), + ('view v = "abc\n"\n', '1:10', 'a string ends on the line it starts'), + ('state a: text\ninput typed = text\nview v = typed\n', '3:10', "'typed' is only available in its handler"), + ('input typed = text\non typed.down {\n}\n', '2:4', "'typed' is a text input; write on typed { }"), + ('view v = text(3)\n', '1:10', 'text(...) converts bytes, not a number'), + ('view v = "é"\nview w = 1 é\n', '2:12', 'unexpected byte 0xc3 outside a string'), ] ACCEPTED = [ @@ -73,6 +96,12 @@ ACCEPTED = [ 'state d: bytes by key\nstate edits = 0\ninput m = message by key { at: u32; text: bytes }\n' 'input peek = message by key\non m {\n d[key].replace(at_or_end .. at_or_end, m.text)\n}\n'.replace('at_or_end', 'min(m.at, d[key].bytes)') + 'on m {\n edits = edits + 1\n require m.text.bytes > 0\n}\non peek { reply d[key] }\n', + # a keyed document loaded into a text state, edited with cursors and saved back as its UTF-8 bytes + 'state documents: bytes by key\nstate note: text\nstate at: cursor in note\n' + 'input open = message by key\ninput save = message by key\ninput typed = text\n' + 'on open {\n note = text(documents[key])\n at = note.end\n}\n' + 'on typed { note.insert(at, typed) }\n' + 'on save {\n documents[key].replace(0 .. documents[key].bytes, note)\n reply "saved {note.chars} characters"\n}\n', ] @@ -291,12 +320,106 @@ def editor_parity(aup, pure, work, seeds, operations=160): return compared, dict(sorted(outcomes.items())) +class NotesModel: + """Reference semantics of examples/notes.aup: positions are Unicode + scalars; every cursor at or after an insertion point moves after it; a + removal gathers cursors inside it; line moves keep the column.""" + + def __init__(self): + self.note, self.at, self.mark = '', 0, 0 + + def line_start(self, i): + return self.note.rfind('\n', 0, i) + 1 + + def line_end(self, i): + end = self.note.find('\n', i) + return len(self.note) if end < 0 else end + + def insert(self, text): + p = self.at + self.note = self.note[:p] + text + self.note[p:] + self.at, self.mark = [c + len(text) if c >= p else c for c in (self.at, self.mark)] + + def remove(self, a, b): + self.note = self.note[:a] + self.note[b:] + self.at, self.mark = [c - (b - a) if c >= b else a if c > a else c for c in (self.at, self.mark)] + + def key(self, name): + n, at = len(self.note), self.at + if name == 'left': self.at = max(0, at - 1) + elif name == 'right': self.at = min(n, at + 1) + elif name in ('up', 'down'): + start = self.line_start(at) + column = at - start + if name == 'up': + if start == 0: self.at = 0 + else: + previous = self.line_start(start - 1) + self.at = previous + min(column, start - 1 - previous) + else: + end = self.line_end(at) + if end == n: self.at = n + else: + self.at = end + 1 + min(column, self.line_end(end + 1) - end - 1) + elif name == 'back': self.remove(max(0, at - 1), at) + elif name == 'forward': self.remove(at, min(n, at + 1)) + elif name == 'home': self.at = 0 + elif name == 'finish': self.at = n + elif name == 'set_mark': self.mark = at + + def expected(self): + byte = lambda i: len(self.note[:i].encode()) + return ({'title': '메모', 'note': self.note, 'at': byte(self.at), 'mark': byte(self.mark)}, + {'header': f'메모 · {len(self.note)}자', + 'current': self.note[self.line_start(self.at):self.line_end(self.at)], + 'position': f"line {self.note[:self.at].count(chr(10)) + 1}, char {self.at}", + 'size': len(self.note.encode()), 'lines': self.note.count('\n') + 1}) + + +def text_parity(aup, pure_run, work, seeds, steps=400): + module = work / 'notes.pmod' + compile_module(aup, ROOT / 'examples/notes.aup', module) + words = ['a', 'Pure', '안녕', '\n', '😀', 'e\u0301', '한글\n줄', 'xy z', '', '\u0000', '👩\u200d💻'] + keys = ['left', 'right', 'up', 'down', 'back', 'forward', 'home', 'finish', 'set_mark'] + compared = 0 + for seed in seeds: + rng = random.Random(seed + 7000) + model, commands, expected = NotesModel(), [], [] + for _ in range(steps): + if rng.random() < 0.45: + word = rng.choice(words) + commands.append('type typed ' + (word.encode().hex() or '-')) + model.insert(word) + else: + name = rng.choice(keys) + commands.append(f'down {name}') + model.key(name) + expected.append(model.expected()) + lines = run(pure_run, module, stdin='\n'.join(commands) + '\n').stdout.splitlines() + assert len(lines) == len(commands), (seed, len(lines)) + for line, command, (states, views) in zip(lines, commands, expected): + snapshot = json.loads(line) + assert snapshot['error'] is None, (seed, command, snapshot) + assert snapshot['states'] == states, (seed, command, snapshot['states'], states) + assert snapshot['views'] == views, (seed, command, snapshot['views'], views) + compared += 1 + # Escapes, interpolation and number text. + source = work / 'strings.aup' + source.write_text('state t: text = "a\\tb\\u{AC00}\\{x\\}\\"q\\"\\\\"\n' + 'view v = "{t}|{1 / 4}|{1e21}|{0.1 + 0.2}|{-0}|{100}|{2.5e-7}|{-1234567.5}"\n') + compile_module(aup, source, work / 'strings.pmod') + shown = json.loads(run(pure_run, work / 'strings.pmod', stdin='show\n').stdout) + assert shown['views']['v'] == 'a\tb가{x}"q"\\|0.25|1e+21|0.30000000000000004|0|100|2.5e-7|-1234567.5', shown + return compared + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--aup', type=Path, required=True) parser.add_argument('--pure-run', type=Path, required=True) parser.add_argument('--pure', type=Path, default=ROOT.parent / 'pure' / 'prototypes' / 'connections') parser.add_argument('--editor-seeds', type=int, default=4) + parser.add_argument('--text-seeds', type=int, default=12) parser.add_argument('--workbench', type=Path, default=ROOT.parent / 'pure' / 'prototypes' / 'workbench') parser.add_argument('--seeds', type=int, default=12) parser.add_argument('--generated', type=int, default=600) @@ -360,6 +483,7 @@ def main(): report['parity_snapshots'] = sum(compared for compared, _ in results) report['parity_failed_events_matched'] = sum(failures for _, failures in results) report['parity_programs'] = len(pairs) + report['text_model_snapshots'] = text_parity(aup, pure_run, work, range(args.text_seeds)) report['editor_parity_operations'], report['editor_parity_outcomes'] = \ editor_parity(aup, args.pure.resolve(), work, range(args.editor_seeds)) print(json.dumps(report)) |
