diff options
Diffstat (limited to 'tests/check.py')
| -rw-r--r-- | tests/check.py | 116 |
1 files changed, 111 insertions, 5 deletions
diff --git a/tests/check.py b/tests/check.py index 1bf107d..83757ed 100644 --- a/tests/check.py +++ b/tests/check.py @@ -10,6 +10,10 @@ counts and event failures as the workbench Zig engine (via its Node CLI). Workbench JSON prints -0 as 0, so signed zeros compare numerically there; Pure's unit tests check signed zeros directly. Other values compare exactly. +5. Editor parity: the document editor written in aup, run by pure-edit-module, + answers random show/edit/apply operations exactly as pure-edit running the + original editor.pure (stdout, stderr, exit code), on the same tree store + format, and each CLI reads the other's store identically. """ import argparse import json @@ -32,7 +36,7 @@ DIAGNOSTICS = [ ('state a = 0\non a.down {\n}\n', '2:4', "'a' is a state, not an input"), ('state a = 0\non frame {\n b = 1\n}\n', '3:3', "unknown state 'b'"), ('state a = 0\nview v = a\non frame {\n v = 1\n}\n', '4:3', "cannot assign to view 'v'"), - ('input k = key Space\nview v = k\n', '2:10', "'k' is a input, not a state"), + ('input k = key Space\nview v = k\n', '2:10', "'k' is a key input, not a state"), ('state a = 0\nview v = min(a)\n', '2:10', 'min expects 2 arguments'), ('state if = 0\n', '1:7', "'if' is reserved"), ('state a = 0 $\n', '1:13', "unexpected character '$'"), @@ -43,6 +47,22 @@ DIAGNOSTICS = [ ('view v = ' + '+'.join(['1'] * 300) + '\n', '1:', 'program exceeds 512 instructions'), ('view v = ' + '(' * 70 + '1' + ')' * 70 + '\n', '1:', 'expression nesting exceeds 64'), ('state s = 0\n' * 1 + ''.join(f'state s{i} = 0\n' for i in range(32)), '33:7', 'at most 32 states are allowed'), + # v0b: messages, keyed documents and types + ('state d: bytes by key\ninput m = message\non m {\n reply d[key]\n}\n', '4:9', 'd[key] needs the handler of a message by key'), + ('input m = message by key { a: u32 }\non m {\n reply m.a\n}\n', '3:9', 'expected bytes, not a number'), + ('input m = message by key {\n b: bytes\n a: u32\n}\n', '3:3', 'a bytes field must be the last field'), + ('input m = message by key\non m.down {\n}\n', '2:4', "'m' is a message; write on m { }"), + ('input k = key Space\non k {\n}\n', '2:4', "'k' is a key input; write on k.down or on k.up"), + ('state d: bytes by key\ninput m = message by key { a: bytes }\non m {\n d[1].replace(0 .. 0, m.a)\n}\n', '4:5', + 'a collection is indexed by the message key'), + ('input m = message by key { a: u32 }\nview v = m.a\n', '2:10', "fields of 'm' are only available in its handler"), + ('state x = 0\ninput m = message by key { a: bytes }\non m {\n x = m.a\n}\n', '4:7', 'expected a number, not bytes'), + ('input m = message by key { a: u32 }\non m {\n reply m.z\n}\n', '3:9', "message 'm' has no field 'z'"), + ('state x = 0\non frame {\n require x.bytes\n}\n', '3:13', '.bytes needs bytes, not a number'), + ('state d: bytes by key\nstate e: bytes by key\n', '2:7', 'at most one keyed collection is allowed'), + ('state d: bytes by key\ninput m = message by key { a: bytes }\non m {\n d[key].replace(0 .. m.a, m.a)\n}\n', '4:23', + 'expected a number, not bytes'), + ('input m = message by key { a: u32 }\non frame {\n reply m.a\n}\n', '3:3', 'reply is only available in a message handler'), ] ACCEPTED = [ @@ -50,6 +70,9 @@ ACCEPTED = [ 'state a = 0\non frame {\n if a > 1 {\n a = 1\n }\n else {\n a = 2\n }\n}\n', 'state a = 0; state b = 1\nview v = clamp(\n a,\n 0,\n b\n)\non frame { a = a + 1; b = b + 2 }\n', '# empty handlers and comments only\ninput k = key KeyZ\non k.down {\n}\non k.up { }\n', + 'state d: bytes by key\nstate edits = 0\ninput m = message by key { at: u32; text: bytes }\n' + 'input peek = message by key\non m {\n d[key].replace(at_or_end .. at_or_end, m.text)\n}\n'.replace('at_or_end', 'min(m.at, d[key].bytes)') + + 'on m {\n edits = edits + 1\n require m.text.bytes > 0\n}\non peek { reply d[key] }\n', ] @@ -58,6 +81,11 @@ def run(*args, stdin=None, check=True): check=check, timeout=60) +def run_bytes(*args): + """Document bytes are arbitrary; compare outputs as bytes.""" + return subprocess.run([str(a) for a in args], capture_output=True, check=False, timeout=60) + + def compile_module(aup, source_path, out): run(aup, 'build', source_path, '-o', out) return out.read_bytes() @@ -162,13 +190,16 @@ def properties(aup, pure_run, work, rounds): """Generated programs compile and verify; damaged sources never crash the compiler, and whatever it accepts, Pure accepts.""" rng = random.Random(2026) - accepted_damaged = 0 + accepted_damaged = over_limit = 0 examples = [path.read_text() for path in sorted(ROOT.glob('examples/*.aup'))] for number in range(rounds): source, inputs = random_program(rng) path, module = work / 'generated.aup', work / 'generated.pmod' path.write_text(source) result = run(aup, 'build', path, '-o', module, check=False) + if result.returncode == 1 and 'program exceeds 512 instructions' in result.stderr: + over_limit += 1 # well formed but longer than a module may be + continue assert result.returncode == 0, (source, result.stderr) commands = ''.join(rng.choice(['frame\n'] + [f'down {i}\nup {i}\n' for i in inputs]) for _ in range(8)) lines = run(pure_run, module, stdin=commands + 'show\n', check=False).stdout.splitlines() @@ -188,13 +219,84 @@ def properties(aup, pure_run, work, rounds): accepted_damaged += 1 shown = run(pure_run, module, stdin='frame\nshow\n', check=False).stdout.splitlines() assert json.loads(shown[-1])['error'] is None or 'event failed' in shown[-1], shown - return accepted_damaged + return accepted_damaged, over_limit + + +def editor_parity(aup, pure, work, seeds, operations=160): + """Differential run of the old and the new editor on separate stores.""" + old, new = pure / 'build/tree/pure-edit', pure / 'build/pure-edit-module' + variants = [('examples/editor.aup', pure / 'examples/editor.pure'), + ('examples/bounded-editor.aup', pure / 'examples/bounded-editor.pure')] + compared, outcomes = 0, {} + for source, program in variants: + module = work / (Path(source).stem + '.pmod') + compile_module(aup, ROOT / source, module) + for seed in seeds: + rng = random.Random(seed * 31 + len(source)) + stores = {'old': work / f'old-{seed}', 'new': work / f'new-{seed}'} + for store in stores.values(): + if store.exists(): + for path in sorted(store.rglob('*'), reverse=True): + path.unlink() if path.is_file() else path.rmdir() + revision, size = {}, {} + keys = ['a', 'b', 'long-key_1'] + for step in range(operations): + key = rng.choice(keys) if rng.random() > 0.03 else 'bad.key' + current, length = revision.get(key, 0), size.get(key, 0) + choice = rng.random() + pick = rng.random() + base = (current if pick < 0.7 else max(current - 1, 0) if pick < 0.8 else 0 if pick < 0.9 + else rng.choice([7, 9223372036854775807])) + budget = ['--budget', str(rng.choice([4, 16]))] if rng.random() < 0.05 else [] + if choice < 0.35: + command = ['show', key] + elif choice < 0.85: + offset = rng.randint(0, length + 3) + remove = rng.choice([0, 0, 1, 2, 5, length]) + body = bytes(rng.randrange(256) for _ in range(rng.choice([0, 1, 5, 50, 5000]))) + insert = work / f'insert-{step}' + insert.write_bytes(body) + command = ['edit', key, str(base), str(offset), str(remove), str(insert)] + else: + if rng.random() < 0.5: + payload = bytes(rng.randrange(256) for _ in range(rng.randint(0, 12))) + else: + payload = (rng.randint(0, length).to_bytes(4, 'little') + + rng.randint(0, 3).to_bytes(4, 'little') + b'raw') + raw = work / f'raw-{step}' + raw.write_bytes(payload) + command = ['apply', key, str(base), str(raw)] + theirs = run_bytes(old, '--program', program, '--store', stores['old'], *budget, *command) + ours = run_bytes(new, '--module', module, '--store', stores['new'], *budget, *command) + where = f'{source} seed {seed} step {step}: {command}' + assert (ours.returncode, ours.stdout, ours.stderr) == (theirs.returncode, theirs.stdout, theirs.stderr), \ + (where, ours, theirs) + compared += 1 + text = theirs.stderr.decode(errors='replace').strip().splitlines() + kind = (json.loads(text[-1]).get('error', 'ok') if text and text[-1].startswith('{') + else text[-1] if text else 'ok') if theirs.returncode else 'ok' + outcomes[f'{command[0]}:{kind}'] = outcomes.get(f'{command[0]}:{kind}', 0) + 1 + report = theirs.stderr if command[0] == 'show' else theirs.stdout + if theirs.returncode == 0: + revision[key] = int(json.loads(report.decode().splitlines()[-1])['revision']) + shown = theirs if command[0] == 'show' else \ + run_bytes(old, '--program', program, '--store', stores['old'], 'show', key) + size[key] = len(shown.stdout) + # Each CLI reads the store the other one wrote. + for key in keys: + across = [run_bytes(old, '--program', program, '--store', stores['new'], 'show', key), + run_bytes(new, '--module', module, '--store', stores['old'], 'show', key), + run_bytes(old, '--program', program, '--store', stores['old'], 'show', key)] + assert len({(r.returncode, r.stdout, r.stderr) for r in across}) == 1, (source, seed, key) + return compared, dict(sorted(outcomes.items())) def main(): parser = argparse.ArgumentParser() parser.add_argument('--aup', type=Path, required=True) parser.add_argument('--pure-run', type=Path, required=True) + parser.add_argument('--pure', type=Path, default=ROOT.parent / 'pure' / 'prototypes' / 'connections') + parser.add_argument('--editor-seeds', type=int, default=4) parser.add_argument('--workbench', type=Path, default=ROOT.parent / 'pure' / 'prototypes' / 'workbench') parser.add_argument('--seeds', type=int, default=12) parser.add_argument('--generated', type=int, default=600) @@ -236,8 +338,10 @@ def main(): final = json.loads(lines[-1]) assert final['states'] == {'bonus': 7, 'count': 9} and final['views'] == {'doubled': 18}, final report['replace_keeps_states_by_name'] = True - report['generated_programs_verified'] = args.generated - report['damaged_sources_accepted_and_verified'] = properties(aup, pure_run, work, args.generated) + accepted_damaged, over_limit = properties(aup, pure_run, work, args.generated) + report['generated_programs_verified'] = args.generated - over_limit + report['generated_programs_over_instruction_limit'] = over_limit + report['damaged_sources_accepted_and_verified'] = accepted_damaged wb = workbench / 'examples' pairs = [ @@ -256,6 +360,8 @@ def main(): report['parity_snapshots'] = sum(compared for compared, _ in results) report['parity_failed_events_matched'] = sum(failures for _, failures in results) report['parity_programs'] = len(pairs) + report['editor_parity_operations'], report['editor_parity_outcomes'] = \ + editor_parity(aup, args.pure.resolve(), work, range(args.editor_seeds)) print(json.dumps(report)) |