140's 5, line breaking (UAX #14, Unicode 16.0), written in aup with no new opcode (aup's examples/linebreak.aup, the properties as data: 2,787 ranges from the UCD 16.0 Vega's text is frozen at, built by Vega's text/unicode.py, loaded once): every line of LineBreakTest.txt (16,672) and 2,000 random lines (their boundaries as linebreak.py, the Python reference, which passes the test whole, works them out) right on the three paths, the paths' outputs the same; module_specs in make check: the ranges and the lines kept in tests/specs/linebreak (derived from the UCD, the Unicode License v3's notice), run by pure-run --views; bars: written, no workaround (three frictions: hex literals always i64, number no let's name, a let's list one a line), 176 lines against Python's 99, 6,920 ns a code point against Vega's C 34.7 (199x), pure Python 2,523 (Pure 2.7x slower), 1,774 instructions a code point (setup 260, the ranges' binary search 360, decide about 1,150 a boundary); the decisions before 8 (box layout: Taffy can be fetched by cargo, a subset to choose first; as a handler) and 10 (a one-shot timer is a core primitive one spec needs: noted, not made); the suite table's row 5; CHECKPOINT's direction row

4 files changed, 46 insertions, 1 deletions

diff --git a/tests/check.py b/tests/check.py
index 046e61a..7a5f50e 100644
--- a/tests/check.py
+++ b/tests/check.py
@@ -1735,7 +1735,8 @@ def module_specs(aup, pure, work):
     messages dnspython made (tests/specs/dns) read by its examples/dns.aup and
     written back by its view to the same records, cut ones refused; HTTP/1.1
     request streams (tests/specs/http) read by its examples/http.aup a packet
-    at a time, cut anywhere, to the same requests."""
+    at a time, cut anywhere, to the same requests; line breaking (UAX #14) by its
+    examples/linebreak.aup on LineBreakTest.txt's lines (tests/specs/linebreak)."""
     import hashlib
     import os
     import random
@@ -1933,6 +1934,34 @@ def module_specs(aup, pure, work):
             assert fed(packets, {})[1] == want, ('http cut', len(packets))
             cuts += 1
     runs['http'] = {'streams': len(made), 'requests': sum(len(w['requests']) for w in reading), 'cuts': cuts}
+    # Line breaking (UAX #14, its examples/linebreak.aup): the properties' ranges loaded (tests/specs/linebreak, from the
+    # UCD 16.0 Vega's text is frozen at), then every line of LineBreakTest.txt and 2,000 random ones, each line's
+    # boundaries as the test says, on the three paths; pure-run prints views alone (the ranges would be in every line).
+    program = work / 'linebreak.pmod'
+    compile_module(aup, EXAMPLES / 'linebreak.aup', program)
+    given, lines = (pure / 'tests/specs/linebreak/lines.bin').read_bytes(), []
+    while given:
+        n = struct.unpack_from('<I', given)[0]
+        lines.append((struct.unpack_from(f'<{n}I', given, 4), [b == 1 for b in given[4 + 4 * n:5 + 5 * n]]))
+        given = given[5 + 5 * n:]
+    commands = work / 'linebreak.commands'
+    with open(commands, 'w') as f:
+        f.write(f"message load @{pure / 'tests/specs/linebreak/ranges.bin'}\n")
+        for codes, _ in lines:
+            f.write(f"message line {b''.join(struct.pack('<d', c) for c in codes).hex() or '-'}\n")
+    shown = {}
+    for name, env in (('batches', {'PURE_JIT': '0'}), ('rounds', {'PURE_JIT': '0', 'PURE_BATCH': '0'}), ('all', {'PURE_JIT': 'all'})):
+        with open(commands) as stdin:
+            shown[name] = subprocess.run([str(pure / 'build/pure-run'), str(program), '--views'], stdin=stdin, capture_output=True,
+                                         text=True, timeout=600, env=dict(os.environ, **env)).stdout
+    for name in ('rounds', 'all'):
+        assert shown[name] == shown['batches'], ('linebreak', name)
+    out = shown['batches'].splitlines()[1:]
+    assert len(out) == len(lines), ('linebreak', len(out))
+    for (codes, want), line in zip(lines, out):
+        state = json.loads(line)
+        assert state['fault'] == 'none' and [b != 0 for b in bytes.fromhex(state['views']['boundaries'])] == want, codes
+    runs['linebreak'] = {'lines': len(lines), 'code_points': sum(len(c) for c, _ in lines)}
     return runs
 
 
diff --git a/tests/specs/linebreak/README.md b/tests/specs/linebreak/README.md
new file mode 100644
index 0000000..9fe2f73
--- /dev/null
+++ b/tests/specs/linebreak/README.md
@@ -0,0 +1,16 @@
+# Line breaking's properties and lines (UAX #14, Unicode 16.0)
+
+`ranges.bin` and `lines.bin` are what `research/experiments/044-specs/linebreak_check.py` writes (its first argument)
+from the Unicode Character Database 16.0.0 that Vega's text is frozen at (`vega/build/upstream/ucd-16.0.0`, read by
+Vega's `text/unicode.py`): Copyright © 1991-2024 Unicode, Inc., under the Unicode License v3
+(https://www.unicode.org/license.txt), which lets data derived from it be copied with this notice.
+
+- `ranges.bin`: the properties' ranges, 2,787 rows of six little-endian binary64s (start, class after LB1 in
+  `linebreak.aup`'s numbering, initial quotation, final quotation, East Asian, pictographic and unassigned), as the app's
+  load message takes them.
+- `lines.bin`: every line of `auxiliary/LineBreakTest.txt` (16,672), then 2,000 random lines (code points drawn from each
+  class, their boundaries as `linebreak.py` works them out). Each is a u32 count, its u32 code points and count + 1
+  bytes, 1 where a line may break.
+
+`tests/check.py`'s `module_specs` loads the ranges into aup's `examples/linebreak.aup` and gives it every line, on Pure's
+three paths.
diff --git a/tests/specs/linebreak/lines.bin b/tests/specs/linebreak/lines.bin
new file mode 100644
index 0000000..04fd783
--- /dev/null
+++ b/tests/specs/linebreak/lines.bin
diff --git a/tests/specs/linebreak/ranges.bin b/tests/specs/linebreak/ranges.bin
new file mode 100644
index 0000000..87d5048
--- /dev/null
+++ b/tests/specs/linebreak/ranges.bin