diff options
| -rw-r--r-- | compiler/compile.c | 38 | ||||
| -rw-r--r-- | docs/design/002-dsl-direction.md | 35 | ||||
| -rw-r--r-- | examples/documents.aup | 39 | ||||
| -rw-r--r-- | tests/check.py | 71 |
4 files changed, 168 insertions, 15 deletions
diff --git a/compiler/compile.c b/compiler/compile.c index 7284414..72e58e1 100644 --- a/compiler/compile.c +++ b/compiler/compile.c @@ -1745,7 +1745,7 @@ static Expr *plain_string(Compiler *c, const char *what) { /* record Name { field = number, field = false, field: text = "..." } */ /* A field's type after ':' (a record's field or a variant's case value): text, i64, number, bool, list of X or X. */ static void declared_type(Compiler *c, Record *r, unsigned index, bool variant) { - Token type = expect_name(c, variant ? "the type of the case's value" : "text, i64, a record, a variant or list of one"); + Token type = expect_name(c, variant ? "the type of the case's value" : "text, bytes, i64, a record, a variant or list of one"); if (!strcmp(type.text, "list")) { expect_word(c, "of"); Token of = expect_name(c, "what the list holds"); @@ -1755,13 +1755,14 @@ static void declared_type(Compiler *c, Record *r, unsigned index, bool variant) return; } if (!strcmp(type.text, "text")) r->declared[index].kind = TEXT; + else if (!strcmp(type.text, "bytes")) r->declared[index].kind = BYTES; /* (Pure 027's 4b) */ else if (!strcmp(type.text, "i64")) r->declared[index].kind = I64; else if (variant && !strcmp(type.text, "number")) r->declared[index].kind = NUM; else if (variant && !strcmp(type.text, "bool")) r->declared[index].kind = BOOL; else { /* a record's or a variant's fields, flattened once all are declared */ if (reserved_name(type.text) || !strcmp(type.text, "number") || !strcmp(type.text, "bool")) fail_at(c, type.line, type.column, "a field is a number (field = 0), a bool (field = false), text (field: text), " - "an i64 (field: i64), a record, a variant or a list of one"); + "bytes (field: bytes), an i64 (field: i64), a record, a variant or a list of one"); memcpy(r->declared[index].type, type.text, NAME); r->typed = true; } @@ -1797,11 +1798,12 @@ static void record_declaration(Compiler *c) { take(c); declared_type(c, r, index, false); if (is(c, "=")) { - if (r->declared[index].kind != TEXT && r->declared[index].kind != I64) + if (r->declared[index].kind != TEXT && r->declared[index].kind != BYTES && r->declared[index].kind != I64) fail_at(c, c->token.line, c->token.column, "a record, a variant or a list in a field begins as its own default"); take(c); r->declared[index].given = true; - if (r->declared[index].kind == TEXT) r->declared[index].string = plain_string(c, "a text field"); + if (r->declared[index].kind == TEXT || r->declared[index].kind == BYTES) + r->declared[index].string = plain_string(c, r->declared[index].kind == TEXT ? "a text field" : "a bytes field"); else r->declared[index].integer = signed_integer(c, "an i64 field's default"); } } else { @@ -3507,7 +3509,13 @@ static Kind value(Compiler *c, const Expr *e, bool handler, int height) { if (dot) { /* item.items.count: a list in a field, then what a list has; item.name.bytes: a text's */ *dot = 0; field = field_named(r, path); - if (field >= 0 && (r->field_kinds[field] == TEXT || r->field_kinds[field] == BYTES)) { + if (field >= 0 && r->field_kinds[field] == BYTES) { /* item.body.bytes: a bytes field's length */ + emit(c, e->line, e->column, ITEM_GET, (uint32_t)item_list | (uint32_t)field << 16, 0); + if (strcmp(dot + 1, "bytes")) fail_at(c, e->line, e->column, "bytes have bytes, not '%s'", dot + 1); + emit(c, e->line, e->column, LEN, 0, 0); + return (Kind){NUM, -1}; + } + if (field >= 0 && r->field_kinds[field] == TEXT) { const char *m = dot + 1; emit(c, e->line, e->column, ITEM_GET, (uint32_t)item_list | (uint32_t)field << 16, 0); if (!strcmp(m, "chars") || !strcmp(m, "lines") || !strcmp(m, "bytes")) { @@ -4270,8 +4278,10 @@ static void record_fields_of(Compiler *c, const Expr *e, const Record *r, const if (values[k]) list_field(c, r, k, values[k], at); else emit(c, e->line, e->column, LIST_VALUE, (uint32_t)hidden_list(c, r->field_record[k], false, e->line, e->column), 0); } else if (values[k]) expect_kind(c, values[k], wanted(c, values[k], c->handling, at, field_kind(r, k)), field_kind(r, k)); - else if (r->field_kinds[k] == TEXT) emit(c, e->line, e->column, TEXT_CONST, r->field_constant[k], 0); - else if (r->field_kinds[k] == I64) emit_integer(c, e->line, e->column, r->field_integer[k]); + else if (r->field_kinds[k] == TEXT || r->field_kinds[k] == BYTES) { + emit(c, e->line, e->column, TEXT_CONST, r->field_constant[k], 0); + if (r->field_kinds[k] == BYTES) emit(c, e->line, e->column, BYTES_FROM, 0, 0); + } else if (r->field_kinds[k] == I64) emit_integer(c, e->line, e->column, r->field_integer[k]); else emit(c, e->line, e->column, CONST, r->field_kinds[k] == BOOL, r->field_number[k]); } } @@ -5374,9 +5384,10 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) { if (*c->state_key[i]) { const Record *r = &c->records[record]; c->list_key[i] = 1 + (int)record_field(c, r, c->state_key[i], c->state_key_line[i], c->state_key_column[i]); - if (r->field_kinds[c->list_key[i] - 1] == BOOL) - fail_at(c, c->state_key_line[i], c->state_key_column[i], "a list is keyed by a number or text; '%s' is a bool", - c->state_key[i]); + int kind = r->field_kinds[c->list_key[i] - 1]; + if (kind == BOOL || kind == BYTES || kind == LIST) + fail_at(c, c->state_key_line[i], c->state_key_column[i], "a list is keyed by a number or text; '%s' is %s", + c->state_key[i], kind == BOOL ? "a bool" : kind == BYTES ? "bytes" : "a list"); } } } @@ -5384,7 +5395,8 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) { Record *r = &c->records[i]; for (unsigned k = 0; k < r->fields; ++k) { Expr empty = {.kind = E_TEXT, .line = r->line, .column = r->column}; - if (r->field_kinds[k] == TEXT) r->field_constant[k] = constant(c, r->field_string[k] ? r->field_string[k] : &empty); + if (r->field_kinds[k] == TEXT || r->field_kinds[k] == BYTES) + r->field_constant[k] = constant(c, r->field_string[k] ? r->field_string[k] : &empty); } } for (unsigned i = 0; i < c->handler_count; ++i) { @@ -5579,9 +5591,9 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) { for (unsigned k = 0; k < c->records[i].fields; ++k) { const Record *r = &c->records[i]; p = put32(p, record_field_name[i][k]); - static const uint32_t types[] = {[NUM] = 0, [TEXT] = 1, [BOOL] = 2, [I64] = 3, [LIST] = 4}; /* Pure's record field types */ + static const uint32_t types[] = {[NUM] = 0, [TEXT] = 1, [BOOL] = 2, [I64] = 3, [LIST] = 4, [BYTES] = 5}; /* Pure's */ p = put32(p, types[r->field_kinds[k]]); - if (r->field_kinds[k] == TEXT) { p = put32(p, r->field_constant[k]); p = put32(p, 0); } + if (r->field_kinds[k] == TEXT || r->field_kinds[k] == BYTES) { p = put32(p, r->field_constant[k]); p = put32(p, 0); } else if (r->field_kinds[k] == I64) p = put64(p, (uint64_t)r->field_integer[k]); else if (r->field_kinds[k] == LIST) p = put64(p, (uint64_t)r->field_record[k]); /* the record of its items */ else p = put_double(p, r->field_number[k]); diff --git a/docs/design/002-dsl-direction.md b/docs/design/002-dsl-direction.md index eb443b9..e3d1568 100644 --- a/docs/design/002-dsl-direction.md +++ b/docs/design/002-dsl-direction.md @@ -817,7 +817,8 @@ on check { ### 저장되는 중첩 (Pure 027의 3단계) -저장 상태와 transient 리스트의 레코드도 리스트를 담는다. 메시지의 행은 아직이다. +저장 상태와 transient 리스트의 레코드도 리스트를 담는다. 메시지의 행도 4a부터 담는다(행은 읽기만 한다: 행 안의 리스트에 +덧붙이는 것은 거절한다). ```text record Card { @@ -847,3 +848,35 @@ on tag { 검증: - 열과 카드와 태그의 앱(`BOARD_SOURCE`)을 무작위 조작 200걸음으로 모델과 대조한다. 메모리에서, 그리고 저장소에서 여덟 번 다시 시작하며(그중 반은 묶음 저장) 돈다. 씨앗 여섯 개에서 모든 걸음의 실패, 상태, 리스트를 읽는 보기가 모델과 같다. + +### 바이트를 담는 필드와 키별 문서 (Pure 027의 4b) + +레코드 필드가 바이트를 담는다(`body: bytes`, 기본값은 `= "…"`의 바이트). 키별 문서(`bytes by key`, `message by key`)가 하던 일을 +키 있는 리스트로 쓴다(`examples/documents.aup`). + +```text +record Doc { + id: text + body: bytes + revision: i64 +} +state docs: list of Doc by id +on edit { + if docs.has(edit.id) { + let d = docs[edit.id] + require d.revision == edit.base # 보낸 쪽이 본 리비전이 아직 그 문서의 것이어야 한다 + require edit.start + edit.remove <= d.body.bytes + docs[edit.id].body = d.body[0 .. edit.start] + edit.insert + d.body[edit.start + edit.remove .. d.body.bytes] + docs[edit.id].revision = d.revision + 1 + } else { + require edit.base == 0 && edit.start == 0 && edit.remove == 0 + docs.add(Doc { id: edit.id, body: edit.insert, revision: 1 }) + } + reply edit.insert +} +``` + +- **뜻.** 바이트 필드는 text 필드처럼 읽고 쓰지만 UTF-8을 묻지 않는다. `x.body.bytes`가 길이다. text는 `bytes(t)`로만 그 자리에 + 간다. +- **검증.** `documents_model`이 무작위 보기와 편집(묵은 리비전, 범위 밖, 아무 바이트, 한 번에 5,000바이트까지) 160걸음을 모델과 + 대조한다. 메모리에서, 그리고 저장소에서 여섯 번 다시 시작하며(반은 묶음 저장) 돈다. 씨앗 넷에서 모든 걸음의 실패와 문서가 같다. diff --git a/examples/documents.aup b/examples/documents.aup new file mode 100644 index 0000000..4671464 --- /dev/null +++ b/examples/documents.aup @@ -0,0 +1,39 @@ +# Revision-checked byte edits of keyed documents. A document is an item of a +# keyed list; an edit names the revision it was made against and commits only +# if that is still the document's (0 for one not there yet), and a failed +# event leaves everything as it was. +record Doc { + id: text + body: bytes + revision: i64 +} +state docs: list of Doc by id + +input show = message { + id: text +} +input edit = message { + id: text + base: i64 + start: u32 + remove: u32 + insert: bytes +} + +on show { + reply docs[show.id].body +} + +on edit { + if docs.has(edit.id) { + let d = docs[edit.id] + require d.revision == edit.base + require edit.start + edit.remove <= d.body.bytes + docs[edit.id].body = d.body[0 .. edit.start] + edit.insert + d.body[edit.start + edit.remove .. d.body.bytes] + docs[edit.id].revision = d.revision + 1 + } else { + require edit.base == 0 && edit.start == 0 && edit.remove == 0 + docs.add(Doc { id: edit.id, body: edit.insert, revision: 1 }) + } + reply edit.insert +} diff --git a/tests/check.py b/tests/check.py index 5a2a462..26cc3f8 100644 --- a/tests/check.py +++ b/tests/check.py @@ -125,6 +125,13 @@ DIAGNOSTICS = [ ' }\n }\n}\n', '16:7', 'an arm after _ is never taken'), (JSON_M + 'state n = 0\non frame {\n let xs: list of Json\n let ms: list of M\n xs.append(Json.arr(ms))\n}\n', '14:22', "'arr' holds a list of Json, not of M"), + # Bytes in a field (Pure 027's 4b): a length alone, a string's bytes as its default, never a text where bytes go, no key. + ('record D {\n b: bytes\n}\nstate ds: list of D\nstate n = 0\non frame {\n for d in ds {\n n = n + d.b.chars\n }\n}\n', + '8:13', "bytes have bytes, not 'chars'"), + ('record D {\n b: bytes = 3\n}\nstate ds: list of D\n', '2:14', 'a bytes field starts from a string'), + ('record D {\n b: bytes\n}\nstate ds: list of D\non frame {\n ds.append(D { b: "x" })\n}\n', '6:20', + 'expected bytes, not text'), + ('record D {\n b: bytes\n}\nstate ds: list of D by b\n', '4:24', "a list is keyed by a number or text; 'b' is bytes"), (JSON_M + 'input m = message {\n docs: list of Json\n}\non m {\n for d in m.docs {\n d.arr.append(Json.null)\n }\n}\n', '15:5', "not of a message's rows or one read from a record"), (JSON_M + 'state n = 0\non frame {\n let ms: list of M\n for m in ms {\n n = m.value\n }\n}\n', '14:9', @@ -237,7 +244,8 @@ DIAGNOSTICS = [ # records, lists and indices ('record T {\n a = 0\n a = 1\n}\n', '3:3', "duplicate field 'a'"), ('record T {\n}\n', '1:8', 'a record needs at least one field'), - ('record T {\n a: number\n}\n', '2:6', 'a field is a number (field = 0), a bool (field = false), text (field: text), an i64 (field: i64), a record'), + ('record T {\n a: number\n}\n', '2:6', + 'a field is a number (field = 0), a bool (field = false), text (field: text), bytes (field: bytes), an i64 (field: i64), a record'), ('record T {\n a = 0\n}\nstate T = 0\n', '4:7', "'T' is already declared as a record"), ('state xs: list of Missing\n', '1:7', "a list holds a record; 'Missing' is not one"), ('state x = 0\nstate at: index of x\n', '2:7', "an index points into a list state; 'x' is not one"), @@ -1771,6 +1779,66 @@ view tags { """ +def documents_model(aup, pure_run, work, seeds, steps=160): + """Pure 027's 4b: keyed byte documents as a keyed list of records holding bytes (examples/documents.aup), each edit + checked against the revision it was made on, run on a store restarting at random points (some runs saving in bursts) + and in memory, against a model: every step's fault and documents alike (bodies of any bytes, up to 5,000 at once), + a failed event leaving nothing.""" + import shutil + import struct + module = work / 'documents.pmod' + compile_module(aup, ROOT / 'examples/documents.aup', module) + ids = ['a', 'b', 'long-key_1', '문서', ''] + compared = restarts = 0 + for seed in seeds: + rng = random.Random(seed + 41000) + docs = {} + commands, expected = [], [] + for _ in range(steps): + i = rng.choice(ids) + body, revision = docs.get(i, (b'', 0)) + fault = 'none' + if rng.random() < 0.25: + commands.append(f"message show {i.encode().hex() or '-'}") + if i not in docs: fault = 'not_found' + else: + pick = rng.random() + base = (revision if pick < 0.7 else revision - 1 if pick < 0.8 else 0 if pick < 0.9 + else rng.choice([7, -1, 9223372036854775807])) + start = rng.randint(0, len(body) + 2) if rng.random() < 0.8 else 0 + remove = rng.choice([0, 0, 1, 2, 5, len(body)]) + insert = bytes(rng.randrange(256) for _ in range(rng.choice([0, 1, 5, 300, 5000]))) + encoded = i.encode() + commands.append('message edit ' + (struct.pack('<IqII', len(encoded), base, start, remove) + encoded + insert).hex()) + if i in docs: + if base != revision or start + remove > len(body): fault = 'limit' + else: docs[i] = (body[:start] + insert + body[start + remove:], revision + 1) + elif base != 0 or start or remove: fault = 'limit' + else: docs[i] = (insert, 1) + expected.append((fault, [{'id': k, 'body': docs[k][0].hex(), 'revision': docs[k][1]} + for k in sorted(docs, key=lambda k: k.encode())])) + runs = {'memory': run(pure_run, module, stdin='\n'.join(commands) + '\n', check=False).stdout.splitlines()} + store = work / f'store-documents-{seed}' + if store.exists(): + shutil.rmtree(store) + cuts, lines = sorted(rng.sample(range(1, steps), 6)), [] + for number_, (start, end) in enumerate(zip([0] + cuts, cuts + [steps])): + batch = ['--batch', str(rng.randint(2, 9))] if number_ % 2 else [] + lines += run(pure_run, module, '--store', store, *batch, stdin='\n'.join(commands[start:end]) + '\n', + check=False).stdout.splitlines() + restarts += 1 + runs['stored'] = lines + for way, outputs in runs.items(): + assert len(outputs) == steps, (way, seed, len(outputs)) + for line, command, (fault, want) in zip(outputs, commands, expected): + snapshot = json.loads(line) + where = (way, seed, command[:40]) + assert snapshot['fault'] == fault, (where, snapshot['fault']) + assert snapshot['states']['docs'] == want, (where, snapshot['states']['docs'], want) + compared += 1 + return [compared, restarts] + + def stored_nested(aup, pure_run, work, seeds, steps=200): """Pure 027's stage 3: a stored list whose items hold lists whose items hold lists (columns of cards of tags), run on a store restarting at random points (some runs saving in bursts) and in memory, against a model: every step's @@ -2499,6 +2567,7 @@ def main(): report['nested_values'] = nested_values(aup, pure_run, work) report['functions'] = functions(aup, pure_run, work) report['stored_nested'] = stored_nested(aup, pure_run, work, range(6)) + report['documents'] = documents_model(aup, pure_run, work, range(4)) report['editor_parity_operations'], report['editor_parity_outcomes'] = \ editor_parity(aup, args.pure.resolve(), work, range(args.editor_seeds)) print(json.dumps(report)) |