Compile messages and keyed documents to module v2

aup now emits Pure module v2 and adds keyed byte collections
(state documents: bytes by key), message inputs by key with u32 and a
trailing bytes field, on-message handlers, field access, name[key] reads
and .bytes sizes, replace(a .. b, value), reply and require, all checked
for number/bytes types with file:line:column diagnostics.

The document editor and its size-bounded variant are rewritten in aup.
make check runs them through pure-edit-module against pure-edit running the
original editor.pure programs: 1,280 random show/edit/apply operations,
including conflicts, limits, missing documents, budgets and invalid keys,
produce byte-identical stdout, stderr and exit codes, and each CLI reads the
other's store identically. v0a parity with the workbench engine, 33
diagnostics and generated/damaged programs still pass, also with a
sanitizer-built compiler.

1 files changed, 111 insertions, 5 deletions

diff --git a/tests/check.py b/tests/check.py
index 1bf107d..83757ed 100644
--- a/tests/check.py
+++ b/tests/check.py
@@ -10,6 +10,10 @@
    counts and event failures as the workbench Zig engine (via its Node CLI).
    Workbench JSON prints -0 as 0, so signed zeros compare numerically there;
    Pure's unit tests check signed zeros directly. Other values compare exactly.
+5. Editor parity: the document editor written in aup, run by pure-edit-module,
+   answers random show/edit/apply operations exactly as pure-edit running the
+   original editor.pure (stdout, stderr, exit code), on the same tree store
+   format, and each CLI reads the other's store identically.
 """
 import argparse
 import json
@@ -32,7 +36,7 @@ DIAGNOSTICS = [
     ('state a = 0\non a.down {\n}\n', '2:4', "'a' is a state, not an input"),
     ('state a = 0\non frame {\n  b = 1\n}\n', '3:3', "unknown state 'b'"),
     ('state a = 0\nview v = a\non frame {\n  v = 1\n}\n', '4:3', "cannot assign to view 'v'"),
-    ('input k = key Space\nview v = k\n', '2:10', "'k' is a input, not a state"),
+    ('input k = key Space\nview v = k\n', '2:10', "'k' is a key input, not a state"),
     ('state a = 0\nview v = min(a)\n', '2:10', 'min expects 2 arguments'),
     ('state if = 0\n', '1:7', "'if' is reserved"),
     ('state a = 0 $\n', '1:13', "unexpected character '$'"),
@@ -43,6 +47,22 @@ DIAGNOSTICS = [
     ('view v = ' + '+'.join(['1'] * 300) + '\n', '1:', 'program exceeds 512 instructions'),
     ('view v = ' + '(' * 70 + '1' + ')' * 70 + '\n', '1:', 'expression nesting exceeds 64'),
     ('state s = 0\n' * 1 + ''.join(f'state s{i} = 0\n' for i in range(32)), '33:7', 'at most 32 states are allowed'),
+    # v0b: messages, keyed documents and types
+    ('state d: bytes by key\ninput m = message\non m {\n  reply d[key]\n}\n', '4:9', 'd[key] needs the handler of a message by key'),
+    ('input m = message by key { a: u32 }\non m {\n  reply m.a\n}\n', '3:9', 'expected bytes, not a number'),
+    ('input m = message by key {\n  b: bytes\n  a: u32\n}\n', '3:3', 'a bytes field must be the last field'),
+    ('input m = message by key\non m.down {\n}\n', '2:4', "'m' is a message; write on m { }"),
+    ('input k = key Space\non k {\n}\n', '2:4', "'k' is a key input; write on k.down or on k.up"),
+    ('state d: bytes by key\ninput m = message by key { a: bytes }\non m {\n  d[1].replace(0 .. 0, m.a)\n}\n', '4:5',
+     'a collection is indexed by the message key'),
+    ('input m = message by key { a: u32 }\nview v = m.a\n', '2:10', "fields of 'm' are only available in its handler"),
+    ('state x = 0\ninput m = message by key { a: bytes }\non m {\n  x = m.a\n}\n', '4:7', 'expected a number, not bytes'),
+    ('input m = message by key { a: u32 }\non m {\n  reply m.z\n}\n', '3:9', "message 'm' has no field 'z'"),
+    ('state x = 0\non frame {\n  require x.bytes\n}\n', '3:13', '.bytes needs bytes, not a number'),
+    ('state d: bytes by key\nstate e: bytes by key\n', '2:7', 'at most one keyed collection is allowed'),
+    ('state d: bytes by key\ninput m = message by key { a: bytes }\non m {\n  d[key].replace(0 .. m.a, m.a)\n}\n', '4:23',
+     'expected a number, not bytes'),
+    ('input m = message by key { a: u32 }\non frame {\n  reply m.a\n}\n', '3:3', 'reply is only available in a message handler'),
 ]
 
 ACCEPTED = [
@@ -50,6 +70,9 @@ ACCEPTED = [
     'state a = 0\non frame {\n  if a > 1 {\n    a = 1\n  }\n  else {\n    a = 2\n  }\n}\n',
     'state a = 0; state b = 1\nview v = clamp(\n  a,\n  0,\n  b\n)\non frame { a = a + 1; b = b + 2 }\n',
     '# empty handlers and comments only\ninput k = key KeyZ\non k.down {\n}\non k.up { }\n',
+    'state d: bytes by key\nstate edits = 0\ninput m = message by key { at: u32; text: bytes }\n'
+    'input peek = message by key\non m {\n  d[key].replace(at_or_end .. at_or_end, m.text)\n}\n'.replace('at_or_end', 'min(m.at, d[key].bytes)')
+    + 'on m {\n  edits = edits + 1\n  require m.text.bytes > 0\n}\non peek { reply d[key] }\n',
 ]
 
 
@@ -58,6 +81,11 @@ def run(*args, stdin=None, check=True):
                           check=check, timeout=60)
 
 
+def run_bytes(*args):
+    """Document bytes are arbitrary; compare outputs as bytes."""
+    return subprocess.run([str(a) for a in args], capture_output=True, check=False, timeout=60)
+
+
 def compile_module(aup, source_path, out):
     run(aup, 'build', source_path, '-o', out)
     return out.read_bytes()
@@ -162,13 +190,16 @@ def properties(aup, pure_run, work, rounds):
     """Generated programs compile and verify; damaged sources never crash the
     compiler, and whatever it accepts, Pure accepts."""
     rng = random.Random(2026)
-    accepted_damaged = 0
+    accepted_damaged = over_limit = 0
     examples = [path.read_text() for path in sorted(ROOT.glob('examples/*.aup'))]
     for number in range(rounds):
         source, inputs = random_program(rng)
         path, module = work / 'generated.aup', work / 'generated.pmod'
         path.write_text(source)
         result = run(aup, 'build', path, '-o', module, check=False)
+        if result.returncode == 1 and 'program exceeds 512 instructions' in result.stderr:
+            over_limit += 1  # well formed but longer than a module may be
+            continue
         assert result.returncode == 0, (source, result.stderr)
         commands = ''.join(rng.choice(['frame\n'] + [f'down {i}\nup {i}\n' for i in inputs]) for _ in range(8))
         lines = run(pure_run, module, stdin=commands + 'show\n', check=False).stdout.splitlines()
@@ -188,13 +219,84 @@ def properties(aup, pure_run, work, rounds):
             accepted_damaged += 1
             shown = run(pure_run, module, stdin='frame\nshow\n', check=False).stdout.splitlines()
             assert json.loads(shown[-1])['error'] is None or 'event failed' in shown[-1], shown
-    return accepted_damaged
+    return accepted_damaged, over_limit
+
+
+def editor_parity(aup, pure, work, seeds, operations=160):
+    """Differential run of the old and the new editor on separate stores."""
+    old, new = pure / 'build/tree/pure-edit', pure / 'build/pure-edit-module'
+    variants = [('examples/editor.aup', pure / 'examples/editor.pure'),
+                ('examples/bounded-editor.aup', pure / 'examples/bounded-editor.pure')]
+    compared, outcomes = 0, {}
+    for source, program in variants:
+        module = work / (Path(source).stem + '.pmod')
+        compile_module(aup, ROOT / source, module)
+        for seed in seeds:
+            rng = random.Random(seed * 31 + len(source))
+            stores = {'old': work / f'old-{seed}', 'new': work / f'new-{seed}'}
+            for store in stores.values():
+                if store.exists():
+                    for path in sorted(store.rglob('*'), reverse=True):
+                        path.unlink() if path.is_file() else path.rmdir()
+            revision, size = {}, {}
+            keys = ['a', 'b', 'long-key_1']
+            for step in range(operations):
+                key = rng.choice(keys) if rng.random() > 0.03 else 'bad.key'
+                current, length = revision.get(key, 0), size.get(key, 0)
+                choice = rng.random()
+                pick = rng.random()
+                base = (current if pick < 0.7 else max(current - 1, 0) if pick < 0.8 else 0 if pick < 0.9
+                        else rng.choice([7, 9223372036854775807]))
+                budget = ['--budget', str(rng.choice([4, 16]))] if rng.random() < 0.05 else []
+                if choice < 0.35:
+                    command = ['show', key]
+                elif choice < 0.85:
+                    offset = rng.randint(0, length + 3)
+                    remove = rng.choice([0, 0, 1, 2, 5, length])
+                    body = bytes(rng.randrange(256) for _ in range(rng.choice([0, 1, 5, 50, 5000])))
+                    insert = work / f'insert-{step}'
+                    insert.write_bytes(body)
+                    command = ['edit', key, str(base), str(offset), str(remove), str(insert)]
+                else:
+                    if rng.random() < 0.5:
+                        payload = bytes(rng.randrange(256) for _ in range(rng.randint(0, 12)))
+                    else:
+                        payload = (rng.randint(0, length).to_bytes(4, 'little') +
+                                   rng.randint(0, 3).to_bytes(4, 'little') + b'raw')
+                    raw = work / f'raw-{step}'
+                    raw.write_bytes(payload)
+                    command = ['apply', key, str(base), str(raw)]
+                theirs = run_bytes(old, '--program', program, '--store', stores['old'], *budget, *command)
+                ours = run_bytes(new, '--module', module, '--store', stores['new'], *budget, *command)
+                where = f'{source} seed {seed} step {step}: {command}'
+                assert (ours.returncode, ours.stdout, ours.stderr) == (theirs.returncode, theirs.stdout, theirs.stderr), \
+                    (where, ours, theirs)
+                compared += 1
+                text = theirs.stderr.decode(errors='replace').strip().splitlines()
+                kind = (json.loads(text[-1]).get('error', 'ok') if text and text[-1].startswith('{')
+                        else text[-1] if text else 'ok') if theirs.returncode else 'ok'
+                outcomes[f'{command[0]}:{kind}'] = outcomes.get(f'{command[0]}:{kind}', 0) + 1
+                report = theirs.stderr if command[0] == 'show' else theirs.stdout
+                if theirs.returncode == 0:
+                    revision[key] = int(json.loads(report.decode().splitlines()[-1])['revision'])
+                    shown = theirs if command[0] == 'show' else \
+                        run_bytes(old, '--program', program, '--store', stores['old'], 'show', key)
+                    size[key] = len(shown.stdout)
+            # Each CLI reads the store the other one wrote.
+            for key in keys:
+                across = [run_bytes(old, '--program', program, '--store', stores['new'], 'show', key),
+                          run_bytes(new, '--module', module, '--store', stores['old'], 'show', key),
+                          run_bytes(old, '--program', program, '--store', stores['old'], 'show', key)]
+                assert len({(r.returncode, r.stdout, r.stderr) for r in across}) == 1, (source, seed, key)
+    return compared, dict(sorted(outcomes.items()))
 
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--aup', type=Path, required=True)
     parser.add_argument('--pure-run', type=Path, required=True)
+    parser.add_argument('--pure', type=Path, default=ROOT.parent / 'pure' / 'prototypes' / 'connections')
+    parser.add_argument('--editor-seeds', type=int, default=4)
     parser.add_argument('--workbench', type=Path, default=ROOT.parent / 'pure' / 'prototypes' / 'workbench')
     parser.add_argument('--seeds', type=int, default=12)
     parser.add_argument('--generated', type=int, default=600)
@@ -236,8 +338,10 @@ def main():
         final = json.loads(lines[-1])
         assert final['states'] == {'bonus': 7, 'count': 9} and final['views'] == {'doubled': 18}, final
         report['replace_keeps_states_by_name'] = True
-        report['generated_programs_verified'] = args.generated
-        report['damaged_sources_accepted_and_verified'] = properties(aup, pure_run, work, args.generated)
+        accepted_damaged, over_limit = properties(aup, pure_run, work, args.generated)
+        report['generated_programs_verified'] = args.generated - over_limit
+        report['generated_programs_over_instruction_limit'] = over_limit
+        report['damaged_sources_accepted_and_verified'] = accepted_damaged
 
         wb = workbench / 'examples'
         pairs = [
@@ -256,6 +360,8 @@ def main():
         report['parity_snapshots'] = sum(compared for compared, _ in results)
         report['parity_failed_events_matched'] = sum(failures for _, failures in results)
         report['parity_programs'] = len(pairs)
+        report['editor_parity_operations'], report['editor_parity_outcomes'] = \
+            editor_parity(aup, args.pure.resolve(), work, range(args.editor_seeds))
     print(json.dumps(report))