Bytes in a record field (Pure 027's 4b-1): body: bytes (= "..." its default's bytes), x.body.bytes its length (LEN), a text going there only as bytes(t); no list keyed by bytes or by a list; examples/documents.aup: keyed byte documents as a keyed list of records, each edit checked against the revision it was made on; documents_model: random shows and edits (stale revisions, past the end, any bytes, up to 5,000 at once) against a model, in memory and on a store restarting six times (half of the runs in bursts), four seeds alike (1,280 steps); four diagnostics

4 files changed, 168 insertions, 15 deletions

diff --git a/compiler/compile.c b/compiler/compile.c
index 7284414..72e58e1 100644
--- a/compiler/compile.c
+++ b/compiler/compile.c
@@ -1745,7 +1745,7 @@ static Expr *plain_string(Compiler *c, const char *what) {
 /* record Name { field = number, field = false, field: text = "..." } */
 /* A field's type after ':' (a record's field or a variant's case value): text, i64, number, bool, list of X or X. */
 static void declared_type(Compiler *c, Record *r, unsigned index, bool variant) {
-    Token type = expect_name(c, variant ? "the type of the case's value" : "text, i64, a record, a variant or list of one");
+    Token type = expect_name(c, variant ? "the type of the case's value" : "text, bytes, i64, a record, a variant or list of one");
     if (!strcmp(type.text, "list")) {
         expect_word(c, "of");
         Token of = expect_name(c, "what the list holds");
@@ -1755,13 +1755,14 @@ static void declared_type(Compiler *c, Record *r, unsigned index, bool variant)
         return;
     }
     if (!strcmp(type.text, "text")) r->declared[index].kind = TEXT;
+    else if (!strcmp(type.text, "bytes")) r->declared[index].kind = BYTES; /* (Pure 027's 4b) */
     else if (!strcmp(type.text, "i64")) r->declared[index].kind = I64;
     else if (variant && !strcmp(type.text, "number")) r->declared[index].kind = NUM;
     else if (variant && !strcmp(type.text, "bool")) r->declared[index].kind = BOOL;
     else { /* a record's or a variant's fields, flattened once all are declared */
         if (reserved_name(type.text) || !strcmp(type.text, "number") || !strcmp(type.text, "bool"))
             fail_at(c, type.line, type.column, "a field is a number (field = 0), a bool (field = false), text (field: text), "
-                    "an i64 (field: i64), a record, a variant or a list of one");
+                    "bytes (field: bytes), an i64 (field: i64), a record, a variant or a list of one");
         memcpy(r->declared[index].type, type.text, NAME);
         r->typed = true;
     }
@@ -1797,11 +1798,12 @@ static void record_declaration(Compiler *c) {
             take(c);
             declared_type(c, r, index, false);
             if (is(c, "=")) {
-                if (r->declared[index].kind != TEXT && r->declared[index].kind != I64)
+                if (r->declared[index].kind != TEXT && r->declared[index].kind != BYTES && r->declared[index].kind != I64)
                     fail_at(c, c->token.line, c->token.column, "a record, a variant or a list in a field begins as its own default");
                 take(c);
                 r->declared[index].given = true;
-                if (r->declared[index].kind == TEXT) r->declared[index].string = plain_string(c, "a text field");
+                if (r->declared[index].kind == TEXT || r->declared[index].kind == BYTES)
+                    r->declared[index].string = plain_string(c, r->declared[index].kind == TEXT ? "a text field" : "a bytes field");
                 else r->declared[index].integer = signed_integer(c, "an i64 field's default");
             }
         } else {
@@ -3507,7 +3509,13 @@ static Kind value(Compiler *c, const Expr *e, bool handler, int height) {
             if (dot) { /* item.items.count: a list in a field, then what a list has; item.name.bytes: a text's */
                 *dot = 0;
                 field = field_named(r, path);
-                if (field >= 0 && (r->field_kinds[field] == TEXT || r->field_kinds[field] == BYTES)) {
+                if (field >= 0 && r->field_kinds[field] == BYTES) { /* item.body.bytes: a bytes field's length */
+                    emit(c, e->line, e->column, ITEM_GET, (uint32_t)item_list | (uint32_t)field << 16, 0);
+                    if (strcmp(dot + 1, "bytes")) fail_at(c, e->line, e->column, "bytes have bytes, not '%s'", dot + 1);
+                    emit(c, e->line, e->column, LEN, 0, 0);
+                    return (Kind){NUM, -1};
+                }
+                if (field >= 0 && r->field_kinds[field] == TEXT) {
                     const char *m = dot + 1;
                     emit(c, e->line, e->column, ITEM_GET, (uint32_t)item_list | (uint32_t)field << 16, 0);
                     if (!strcmp(m, "chars") || !strcmp(m, "lines") || !strcmp(m, "bytes")) {
@@ -4270,8 +4278,10 @@ static void record_fields_of(Compiler *c, const Expr *e, const Record *r, const
             if (values[k]) list_field(c, r, k, values[k], at);
             else emit(c, e->line, e->column, LIST_VALUE, (uint32_t)hidden_list(c, r->field_record[k], false, e->line, e->column), 0);
         } else if (values[k]) expect_kind(c, values[k], wanted(c, values[k], c->handling, at, field_kind(r, k)), field_kind(r, k));
-        else if (r->field_kinds[k] == TEXT) emit(c, e->line, e->column, TEXT_CONST, r->field_constant[k], 0);
-        else if (r->field_kinds[k] == I64) emit_integer(c, e->line, e->column, r->field_integer[k]);
+        else if (r->field_kinds[k] == TEXT || r->field_kinds[k] == BYTES) {
+            emit(c, e->line, e->column, TEXT_CONST, r->field_constant[k], 0);
+            if (r->field_kinds[k] == BYTES) emit(c, e->line, e->column, BYTES_FROM, 0, 0);
+        } else if (r->field_kinds[k] == I64) emit_integer(c, e->line, e->column, r->field_integer[k]);
         else emit(c, e->line, e->column, CONST, r->field_kinds[k] == BOOL, r->field_number[k]);
     }
 }
@@ -5374,9 +5384,10 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) {
             if (*c->state_key[i]) {
                 const Record *r = &c->records[record];
                 c->list_key[i] = 1 + (int)record_field(c, r, c->state_key[i], c->state_key_line[i], c->state_key_column[i]);
-                if (r->field_kinds[c->list_key[i] - 1] == BOOL)
-                    fail_at(c, c->state_key_line[i], c->state_key_column[i], "a list is keyed by a number or text; '%s' is a bool",
-                            c->state_key[i]);
+                int kind = r->field_kinds[c->list_key[i] - 1];
+                if (kind == BOOL || kind == BYTES || kind == LIST)
+                    fail_at(c, c->state_key_line[i], c->state_key_column[i], "a list is keyed by a number or text; '%s' is %s",
+                            c->state_key[i], kind == BOOL ? "a bool" : kind == BYTES ? "bytes" : "a list");
             }
         }
     }
@@ -5384,7 +5395,8 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) {
         Record *r = &c->records[i];
         for (unsigned k = 0; k < r->fields; ++k) {
             Expr empty = {.kind = E_TEXT, .line = r->line, .column = r->column};
-            if (r->field_kinds[k] == TEXT) r->field_constant[k] = constant(c, r->field_string[k] ? r->field_string[k] : &empty);
+            if (r->field_kinds[k] == TEXT || r->field_kinds[k] == BYTES)
+                r->field_constant[k] = constant(c, r->field_string[k] ? r->field_string[k] : &empty);
         }
     }
     for (unsigned i = 0; i < c->handler_count; ++i) {
@@ -5579,9 +5591,9 @@ static void generate(Compiler *c, unsigned char **module, size_t *size) {
         for (unsigned k = 0; k < c->records[i].fields; ++k) {
             const Record *r = &c->records[i];
             p = put32(p, record_field_name[i][k]);
-            static const uint32_t types[] = {[NUM] = 0, [TEXT] = 1, [BOOL] = 2, [I64] = 3, [LIST] = 4}; /* Pure's record field types */
+            static const uint32_t types[] = {[NUM] = 0, [TEXT] = 1, [BOOL] = 2, [I64] = 3, [LIST] = 4, [BYTES] = 5}; /* Pure's */
             p = put32(p, types[r->field_kinds[k]]);
-            if (r->field_kinds[k] == TEXT) { p = put32(p, r->field_constant[k]); p = put32(p, 0); }
+            if (r->field_kinds[k] == TEXT || r->field_kinds[k] == BYTES) { p = put32(p, r->field_constant[k]); p = put32(p, 0); }
             else if (r->field_kinds[k] == I64) p = put64(p, (uint64_t)r->field_integer[k]);
             else if (r->field_kinds[k] == LIST) p = put64(p, (uint64_t)r->field_record[k]); /* the record of its items */
             else p = put_double(p, r->field_number[k]);
diff --git a/docs/design/002-dsl-direction.md b/docs/design/002-dsl-direction.md
index eb443b9..e3d1568 100644
--- a/docs/design/002-dsl-direction.md
+++ b/docs/design/002-dsl-direction.md
@@ -817,7 +817,8 @@ on check {
 
 ### 저장되는 중첩 (Pure 027의 3단계)
 
-저장 상태와 transient 리스트의 레코드도 리스트를 담는다. 메시지의 행은 아직이다.
+저장 상태와 transient 리스트의 레코드도 리스트를 담는다. 메시지의 행도 4a부터 담는다(행은 읽기만 한다: 행 안의 리스트에
+덧붙이는 것은 거절한다).
 
 ```text
 record Card {
@@ -847,3 +848,35 @@ on tag {
 검증:
 - 열과 카드와 태그의 앱(`BOARD_SOURCE`)을 무작위 조작 200걸음으로 모델과 대조한다. 메모리에서, 그리고 저장소에서 여덟 번 다시
   시작하며(그중 반은 묶음 저장) 돈다. 씨앗 여섯 개에서 모든 걸음의 실패, 상태, 리스트를 읽는 보기가 모델과 같다.
+
+### 바이트를 담는 필드와 키별 문서 (Pure 027의 4b)
+
+레코드 필드가 바이트를 담는다(`body: bytes`, 기본값은 `= "…"`의 바이트). 키별 문서(`bytes by key`, `message by key`)가 하던 일을
+키 있는 리스트로 쓴다(`examples/documents.aup`).
+
+```text
+record Doc {
+  id: text
+  body: bytes
+  revision: i64
+}
+state docs: list of Doc by id
+on edit {
+  if docs.has(edit.id) {
+    let d = docs[edit.id]
+    require d.revision == edit.base       # 보낸 쪽이 본 리비전이 아직 그 문서의 것이어야 한다
+    require edit.start + edit.remove <= d.body.bytes
+    docs[edit.id].body = d.body[0 .. edit.start] + edit.insert + d.body[edit.start + edit.remove .. d.body.bytes]
+    docs[edit.id].revision = d.revision + 1
+  } else {
+    require edit.base == 0 && edit.start == 0 && edit.remove == 0
+    docs.add(Doc { id: edit.id, body: edit.insert, revision: 1 })
+  }
+  reply edit.insert
+}
+```
+
+- **뜻.** 바이트 필드는 text 필드처럼 읽고 쓰지만 UTF-8을 묻지 않는다. `x.body.bytes`가 길이다. text는 `bytes(t)`로만 그 자리에
+  간다.
+- **검증.** `documents_model`이 무작위 보기와 편집(묵은 리비전, 범위 밖, 아무 바이트, 한 번에 5,000바이트까지) 160걸음을 모델과
+  대조한다. 메모리에서, 그리고 저장소에서 여섯 번 다시 시작하며(반은 묶음 저장) 돈다. 씨앗 넷에서 모든 걸음의 실패와 문서가 같다.
diff --git a/examples/documents.aup b/examples/documents.aup
new file mode 100644
index 0000000..4671464
--- /dev/null
+++ b/examples/documents.aup
@@ -0,0 +1,39 @@
+# Revision-checked byte edits of keyed documents. A document is an item of a
+# keyed list; an edit names the revision it was made against and commits only
+# if that is still the document's (0 for one not there yet), and a failed
+# event leaves everything as it was.
+record Doc {
+  id: text
+  body: bytes
+  revision: i64
+}
+state docs: list of Doc by id
+
+input show = message {
+  id: text
+}
+input edit = message {
+  id: text
+  base: i64
+  start: u32
+  remove: u32
+  insert: bytes
+}
+
+on show {
+  reply docs[show.id].body
+}
+
+on edit {
+  if docs.has(edit.id) {
+    let d = docs[edit.id]
+    require d.revision == edit.base
+    require edit.start + edit.remove <= d.body.bytes
+    docs[edit.id].body = d.body[0 .. edit.start] + edit.insert + d.body[edit.start + edit.remove .. d.body.bytes]
+    docs[edit.id].revision = d.revision + 1
+  } else {
+    require edit.base == 0 && edit.start == 0 && edit.remove == 0
+    docs.add(Doc { id: edit.id, body: edit.insert, revision: 1 })
+  }
+  reply edit.insert
+}
diff --git a/tests/check.py b/tests/check.py
index 5a2a462..26cc3f8 100644
--- a/tests/check.py
+++ b/tests/check.py
@@ -125,6 +125,13 @@ DIAGNOSTICS = [
      '    }\n  }\n}\n', '16:7', 'an arm after _ is never taken'),
     (JSON_M + 'state n = 0\non frame {\n  let xs: list of Json\n  let ms: list of M\n  xs.append(Json.arr(ms))\n}\n', '14:22',
      "'arr' holds a list of Json, not of M"),
+    # Bytes in a field (Pure 027's 4b): a length alone, a string's bytes as its default, never a text where bytes go, no key.
+    ('record D {\n  b: bytes\n}\nstate ds: list of D\nstate n = 0\non frame {\n  for d in ds {\n    n = n + d.b.chars\n  }\n}\n',
+     '8:13', "bytes have bytes, not 'chars'"),
+    ('record D {\n  b: bytes = 3\n}\nstate ds: list of D\n', '2:14', 'a bytes field starts from a string'),
+    ('record D {\n  b: bytes\n}\nstate ds: list of D\non frame {\n  ds.append(D { b: "x" })\n}\n', '6:20',
+     'expected bytes, not text'),
+    ('record D {\n  b: bytes\n}\nstate ds: list of D by b\n', '4:24', "a list is keyed by a number or text; 'b' is bytes"),
     (JSON_M + 'input m = message {\n  docs: list of Json\n}\non m {\n  for d in m.docs {\n    d.arr.append(Json.null)\n  }\n}\n',
      '15:5', "not of a message's rows or one read from a record"),
     (JSON_M + 'state n = 0\non frame {\n  let ms: list of M\n  for m in ms {\n    n = m.value\n  }\n}\n', '14:9',
@@ -237,7 +244,8 @@ DIAGNOSTICS = [
     # records, lists and indices
     ('record T {\n  a = 0\n  a = 1\n}\n', '3:3', "duplicate field 'a'"),
     ('record T {\n}\n', '1:8', 'a record needs at least one field'),
-    ('record T {\n  a: number\n}\n', '2:6', 'a field is a number (field = 0), a bool (field = false), text (field: text), an i64 (field: i64), a record'),
+    ('record T {\n  a: number\n}\n', '2:6',
+     'a field is a number (field = 0), a bool (field = false), text (field: text), bytes (field: bytes), an i64 (field: i64), a record'),
     ('record T {\n  a = 0\n}\nstate T = 0\n', '4:7', "'T' is already declared as a record"),
     ('state xs: list of Missing\n', '1:7', "a list holds a record; 'Missing' is not one"),
     ('state x = 0\nstate at: index of x\n', '2:7', "an index points into a list state; 'x' is not one"),
@@ -1771,6 +1779,66 @@ view tags {
 """
 
 
+def documents_model(aup, pure_run, work, seeds, steps=160):
+    """Pure 027's 4b: keyed byte documents as a keyed list of records holding bytes (examples/documents.aup), each edit
+    checked against the revision it was made on, run on a store restarting at random points (some runs saving in bursts)
+    and in memory, against a model: every step's fault and documents alike (bodies of any bytes, up to 5,000 at once),
+    a failed event leaving nothing."""
+    import shutil
+    import struct
+    module = work / 'documents.pmod'
+    compile_module(aup, ROOT / 'examples/documents.aup', module)
+    ids = ['a', 'b', 'long-key_1', '문서', '']
+    compared = restarts = 0
+    for seed in seeds:
+        rng = random.Random(seed + 41000)
+        docs = {}
+        commands, expected = [], []
+        for _ in range(steps):
+            i = rng.choice(ids)
+            body, revision = docs.get(i, (b'', 0))
+            fault = 'none'
+            if rng.random() < 0.25:
+                commands.append(f"message show {i.encode().hex() or '-'}")
+                if i not in docs: fault = 'not_found'
+            else:
+                pick = rng.random()
+                base = (revision if pick < 0.7 else revision - 1 if pick < 0.8 else 0 if pick < 0.9
+                        else rng.choice([7, -1, 9223372036854775807]))
+                start = rng.randint(0, len(body) + 2) if rng.random() < 0.8 else 0
+                remove = rng.choice([0, 0, 1, 2, 5, len(body)])
+                insert = bytes(rng.randrange(256) for _ in range(rng.choice([0, 1, 5, 300, 5000])))
+                encoded = i.encode()
+                commands.append('message edit ' + (struct.pack('<IqII', len(encoded), base, start, remove) + encoded + insert).hex())
+                if i in docs:
+                    if base != revision or start + remove > len(body): fault = 'limit'
+                    else: docs[i] = (body[:start] + insert + body[start + remove:], revision + 1)
+                elif base != 0 or start or remove: fault = 'limit'
+                else: docs[i] = (insert, 1)
+            expected.append((fault, [{'id': k, 'body': docs[k][0].hex(), 'revision': docs[k][1]}
+                                     for k in sorted(docs, key=lambda k: k.encode())]))
+        runs = {'memory': run(pure_run, module, stdin='\n'.join(commands) + '\n', check=False).stdout.splitlines()}
+        store = work / f'store-documents-{seed}'
+        if store.exists():
+            shutil.rmtree(store)
+        cuts, lines = sorted(rng.sample(range(1, steps), 6)), []
+        for number_, (start, end) in enumerate(zip([0] + cuts, cuts + [steps])):
+            batch = ['--batch', str(rng.randint(2, 9))] if number_ % 2 else []
+            lines += run(pure_run, module, '--store', store, *batch, stdin='\n'.join(commands[start:end]) + '\n',
+                         check=False).stdout.splitlines()
+            restarts += 1
+        runs['stored'] = lines
+        for way, outputs in runs.items():
+            assert len(outputs) == steps, (way, seed, len(outputs))
+            for line, command, (fault, want) in zip(outputs, commands, expected):
+                snapshot = json.loads(line)
+                where = (way, seed, command[:40])
+                assert snapshot['fault'] == fault, (where, snapshot['fault'])
+                assert snapshot['states']['docs'] == want, (where, snapshot['states']['docs'], want)
+                compared += 1
+    return [compared, restarts]
+
+
 def stored_nested(aup, pure_run, work, seeds, steps=200):
     """Pure 027's stage 3: a stored list whose items hold lists whose items hold lists (columns of cards of tags), run
     on a store restarting at random points (some runs saving in bursts) and in memory, against a model: every step's
@@ -2499,6 +2567,7 @@ def main():
         report['nested_values'] = nested_values(aup, pure_run, work)
         report['functions'] = functions(aup, pure_run, work)
         report['stored_nested'] = stored_nested(aup, pure_run, work, range(6))
+        report['documents'] = documents_model(aup, pure_run, work, range(4))
         report['editor_parity_operations'], report['editor_parity_outcomes'] = \
             editor_parity(aup, args.pure.resolve(), work, range(args.editor_seeds))
     print(json.dumps(report))