From c545227f2461e92afc38a0e1317a288e8c653cc6 Mon Sep 17 00:00:00 2001
From: eunhhu <necpition205@gmail.com>
Date: Mon, 19 Oct 2026 05:42:00 +0100
Subject: 140's 5, line breaking (UAX #14, Unicode 16.0), written in aup with
 no new opcode (aup's examples/linebreak.aup, the properties as data: 2,787
 ranges from the UCD 16.0 Vega's text is frozen at, built by Vega's
 text/unicode.py, loaded once): every line of LineBreakTest.txt (16,672) and
 2,000 random lines (their boundaries as linebreak.py, the Python reference,
 which passes the test whole, works them out) right on the three paths, the
 paths' outputs the same; module_specs in make check: the ranges and the lines
 kept in tests/specs/linebreak (derived from the UCD, the Unicode License v3's
 notice), run by pure-run --views; bars: written, no workaround (three
 frictions: hex literals always i64, number no let's name, a let's list one a
 line), 176 lines against Python's 99, 6,920 ns a code point against Vega's C
 34.7 (199x), pure Python 2,523 (Pure 2.7x slower), 1,774 instructions a code
 point (setup 260, the ranges' binary search 360, decide about 1,150 a
 boundary); the decisions before 8 (box layout: Taffy can be fetched by cargo,
 a subset to choose first; as a handler) and 10 (a one-shot timer is a core
 primitive one spec needs: noted, not made); the suite table's row 5;
 CHECKPOINT's direction row

---
 tests/check.py                   |  31 ++++++++++++++++++++++++++++++-
 tests/specs/linebreak/README.md  |  16 ++++++++++++++++
 tests/specs/linebreak/lines.bin  | Bin 0 -> 399785 bytes
 tests/specs/linebreak/ranges.bin | Bin 0 -> 133776 bytes
 4 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 tests/specs/linebreak/README.md
 create mode 100644 tests/specs/linebreak/lines.bin
 create mode 100644 tests/specs/linebreak/ranges.bin

(limited to 'tests')

diff --git a/tests/check.py b/tests/check.py
index 046e61a..7a5f50e 100644
--- a/tests/check.py
+++ b/tests/check.py
@@ -1735,7 +1735,8 @@ def module_specs(aup, pure, work):
     messages dnspython made (tests/specs/dns) read by its examples/dns.aup and
     written back by its view to the same records, cut ones refused; HTTP/1.1
     request streams (tests/specs/http) read by its examples/http.aup a packet
-    at a time, cut anywhere, to the same requests."""
+    at a time, cut anywhere, to the same requests; line breaking (UAX #14) by its
+    examples/linebreak.aup on LineBreakTest.txt's lines (tests/specs/linebreak)."""
     import hashlib
     import os
     import random
@@ -1933,6 +1934,34 @@ def module_specs(aup, pure, work):
             assert fed(packets, {})[1] == want, ('http cut', len(packets))
             cuts += 1
     runs['http'] = {'streams': len(made), 'requests': sum(len(w['requests']) for w in reading), 'cuts': cuts}
+    # Line breaking (UAX #14, its examples/linebreak.aup): the properties' ranges loaded (tests/specs/linebreak, from the
+    # UCD 16.0 Vega's text is frozen at), then every line of LineBreakTest.txt and 2,000 random ones, each line's
+    # boundaries as the test says, on the three paths; pure-run prints views alone (the ranges would be in every line).
+    program = work / 'linebreak.pmod'
+    compile_module(aup, EXAMPLES / 'linebreak.aup', program)
+    given, lines = (pure / 'tests/specs/linebreak/lines.bin').read_bytes(), []
+    while given:
+        n = struct.unpack_from('<I', given)[0]
+        lines.append((struct.unpack_from(f'<{n}I', given, 4), [b == 1 for b in given[4 + 4 * n:5 + 5 * n]]))
+        given = given[5 + 5 * n:]
+    commands = work / 'linebreak.commands'
+    with open(commands, 'w') as f:
+        f.write(f"message load @{pure / 'tests/specs/linebreak/ranges.bin'}\n")
+        for codes, _ in lines:
+            f.write(f"message line {b''.join(struct.pack('<d', c) for c in codes).hex() or '-'}\n")
+    shown = {}
+    for name, env in (('batches', {'PURE_JIT': '0'}), ('rounds', {'PURE_JIT': '0', 'PURE_BATCH': '0'}), ('all', {'PURE_JIT': 'all'})):
+        with open(commands) as stdin:
+            shown[name] = subprocess.run([str(pure / 'build/pure-run'), str(program), '--views'], stdin=stdin, capture_output=True,
+                                         text=True, timeout=600, env=dict(os.environ, **env)).stdout
+    for name in ('rounds', 'all'):
+        assert shown[name] == shown['batches'], ('linebreak', name)
+    out = shown['batches'].splitlines()[1:]
+    assert len(out) == len(lines), ('linebreak', len(out))
+    for (codes, want), line in zip(lines, out):
+        state = json.loads(line)
+        assert state['fault'] == 'none' and [b != 0 for b in bytes.fromhex(state['views']['boundaries'])] == want, codes
+    runs['linebreak'] = {'lines': len(lines), 'code_points': sum(len(c) for c, _ in lines)}
     return runs
 
 
diff --git a/tests/specs/linebreak/README.md b/tests/specs/linebreak/README.md
new file mode 100644
index 0000000..9fe2f73
--- /dev/null
+++ b/tests/specs/linebreak/README.md
@@ -0,0 +1,16 @@
+# Line breaking's properties and lines (UAX #14, Unicode 16.0)
+
+`ranges.bin` and `lines.bin` are what `research/experiments/044-specs/linebreak_check.py` writes (its first argument)
+from the Unicode Character Database 16.0.0 that Vega's text is frozen at (`vega/build/upstream/ucd-16.0.0`, read by
+Vega's `text/unicode.py`): Copyright © 1991-2024 Unicode, Inc., under the Unicode License v3
+(https://www.unicode.org/license.txt), which lets data derived from it be copied with this notice.
+
+- `ranges.bin`: the properties' ranges, 2,787 rows of six little-endian binary64s (start, class after LB1 in
+  `linebreak.aup`'s numbering, initial quotation, final quotation, East Asian, pictographic and unassigned), as the app's
+  load message takes them.
+- `lines.bin`: every line of `auxiliary/LineBreakTest.txt` (16,672), then 2,000 random lines (code points drawn from each
+  class, their boundaries as `linebreak.py` works them out). Each is a u32 count, its u32 code points and count + 1
+  bytes, 1 where a line may break.
+
+`tests/check.py`'s `module_specs` loads the ranges into aup's `examples/linebreak.aup` and gives it every line, on Pure's
+three paths.
diff --git a/tests/specs/linebreak/lines.bin b/tests/specs/linebreak/lines.bin
new file mode 100644
index 0000000..04fd783
Binary files /dev/null and b/tests/specs/linebreak/lines.bin differ
diff --git a/tests/specs/linebreak/ranges.bin b/tests/specs/linebreak/ranges.bin
new file mode 100644
index 0000000..87d5048
Binary files /dev/null and b/tests/specs/linebreak/ranges.bin differ
-- 
cgit v1.2.3