1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
#!/usr/bin/env python3
"""Research 140: line breaking (UAX #14, Unicode 16.0) written in aup (aup's examples/linebreak.aup).
Builds the properties' table from the UCD Vega's text is frozen at (its text/unicode.py reads the derived files): ranges
of code points, each its class after LB1, initial or final quotation (Pi, Pf), East Asian (F, W, H), and pictographic and
unassigned (Extended_Pictographic and Cn). Loads it into the app (a load message's rows), then gives it every line of
LineBreakTest.txt and random lines (code points drawn from every class), a line a message, on Pure's three paths
(batches, rounds, all compiled): each line's boundaries must be the test's (the random lines': linebreak.py's, which
passes the test whole), the paths the same. pure-run prints views alone (--views): the table would be in every line.
Then counts lines (code lines: neither blank nor a comment; brace lines: a closing brace alone) of linebreak.aup and of
linebreak.py. One JSON line; given a directory, it writes the table and the lines there for linebreak_time.c.
python3 linebreak_check.py [DIRECTORY] [LINES] (LINES: the test's lines to take, all by default)"""
import json, os, random, struct, subprocess, sys, tempfile
from pathlib import Path
here = os.path.dirname(os.path.abspath(__file__))
pure = os.path.normpath(os.path.join(here, '../../..'))
aup = os.environ.get('AUP', os.path.join(pure, '..', 'aup'))
vega = os.environ.get('VEGA', os.path.join(pure, '..', 'vega'))
ucd_path = Path(vega) / 'build/upstream/ucd-16.0.0'
sys.path.insert(0, here)
import linebreak # noqa: E402
CLASSES = 'BK CR LF NL SP ZW ZWJ CM WJ GL BA BB B2 HY CB CL CP EX IN NS OP QU IS NU PO PR SY AK AL AP AS EB EM H2 H3 HL ID JL JV JT RI VF VI'.split()
def lines(path, stop=None):
code = braces = 0
for line in open(path):
if stop and line.startswith(stop): break
text = line.strip()
if not text or text.startswith('#'): continue
code += 1
braces += text == '}'
return code, braces
def table():
"""The ranges' starts and their rows (class, pi, pf, east, pictographic), every code point in one."""
argv, sys.argv = sys.argv, ['unicode.py', str(ucd_path)]
sys.path.insert(0, os.path.join(vega, 'text'))
import unicode as ucd
sys.argv = argv
line = ucd.derived('extracted/DerivedLineBreak.txt', ucd.LINE, ucd.aliases('lb'))
category = ucd.derived('extracted/DerivedGeneralCategory.txt', ucd.CATEGORIES)
width = ucd.derived('extracted/DerivedEastAsianWidth.txt', ucd.WIDTHS, ucd.aliases('ea'))
pictographic = bytearray(ucd.LAST + 1)
for first, last, fields, missing in ucd.ranges('emoji/emoji-data.txt'):
if not missing and fields[0] == 'Extended_Pictographic':
pictographic[first:last + 1] = b'\1' * (last - first + 1)
starts, rows = [], []
for c in range(ucd.LAST + 1):
name, g = ucd.LINE[line[c]], ucd.CATEGORIES[category[c]]
if name in ('AI', 'SG', 'XX'): name = 'AL' # LB1
elif name == 'SA': name = 'CM' if g in ('Mn', 'Mc') else 'AL'
elif name == 'CJ': name = 'NS'
row = (CLASSES.index(name), g == 'Pi', g == 'Pf', ucd.WIDTHS[width[c]] in ('F', 'W', 'H'), bool(pictographic[c]) and g == 'Cn')
if not rows or rows[-1] != row:
starts.append(c)
rows.append(row)
return starts, rows
def test_lines(limit=None):
out = []
for text in open(ucd_path / 'auxiliary/LineBreakTest.txt', encoding='utf-8'):
body = text.split('#', 1)[0].split()
if body:
out.append(([int(x, 16) for x in body if x not in ('÷', '×')], [x == '÷' for x in body if x in ('÷', '×')]))
return out[:limit] if limit else out
def random_lines(chooser, starts, rows, count):
"""Lines of code points drawn from each class's ranges (and from the specials the rules name), their boundaries
as linebreak.py works them out."""
by_class = {}
for k, (start, row) in enumerate(zip(starts, rows)):
end = starts[k + 1] if k + 1 < len(starts) else 0x110000
by_class.setdefault(row[0], []).append((start, end))
specials = [0x2010, 0x25CC, 0x0020, 0x000A, 0x000D, 0x200D, 0x00AB, 0x00BB, 0x201C, 0x201D, 0x1F1E6, 0x1F3FB, 0x1F466]
out = []
for _ in range(count):
codes = []
for _ in range(chooser.randrange(1, 9)):
if chooser.random() < 0.15:
codes.append(chooser.choice(specials))
else:
start, end = chooser.choice(by_class[chooser.choice(list(by_class))])
codes.append(chooser.randrange(start, end))
out.append((codes, [b != linebreak.NONE for b in linebreak.breaks(codes, starts, rows)]))
return out
def run(module, load, given, env):
"""Each line's boundaries as the app gives them (None where its event failed), pure-run's output streamed."""
commands = load + '.commands'
with open(commands, 'w') as f:
f.write(f'message load @{load}\n')
for codes, _ in given:
f.write(f"message line {b''.join(struct.pack('<d', c) for c in codes).hex() or '-'}\n")
with open(commands) as stdin:
process = subprocess.Popen([os.path.join(pure, 'build', 'pure-run'), module, '--views'], stdin=stdin,
stdout=subprocess.PIPE, text=True, env=dict(os.environ, **env))
got, digest = [], 0
for k, line in enumerate(process.stdout):
digest = hash((digest, line))
if k == 0: continue # the load
state = json.loads(line)
got.append([b != 0 for b in bytes.fromhex(state['views']['boundaries'])] if state['fault'] == 'none' else None)
process.wait()
return got, digest
if __name__ == '__main__':
starts, rows = table()
for codes, want in test_lines(): # the reference passes the test whole first
assert [b != linebreak.NONE for b in linebreak.breaks(codes, starts, rows)] == want, codes
given = test_lines(int(sys.argv[2]) if len(sys.argv) > 2 else None)
given += random_lines(random.Random(140), starts, rows, 2000)
with tempfile.TemporaryDirectory() as work:
load = os.path.join(work, 'ranges.bin')
with open(load, 'wb') as f:
f.write(b''.join(struct.pack('<6d', start, *row) for start, row in zip(starts, rows)))
if len(sys.argv) > 1 and sys.argv[1]:
os.makedirs(sys.argv[1], exist_ok=True)
with open(os.path.join(sys.argv[1], 'ranges.bin'), 'wb') as f, open(load, 'rb') as g:
f.write(g.read())
with open(os.path.join(sys.argv[1], 'lines.bin'), 'wb') as f: # each: u32 count, its code points, count + 1 bytes
for codes, want in given:
f.write(struct.pack('<I', len(codes)) + struct.pack(f'<{len(codes)}I', *codes) + bytes(want))
module = os.path.join(work, 'linebreak.pmod')
subprocess.run([os.path.join(aup, 'build', 'aup'), 'build', os.path.join(aup, 'examples', 'linebreak.aup'), '-o', module],
check=True)
right, digests = {}, set()
for name, env in (('batches', {'PURE_JIT': '0'}), ('rounds', {'PURE_JIT': '0', 'PURE_BATCH': '0'}), ('all', {'PURE_JIT': 'all'})):
got, digest = run(module, load, given, env)
right[name] = sum(g == want for g, (_, want) in zip(got, given)) if len(got) == len(given) else 0
digests.add(digest)
tested = len(given) - 2000
aup_code, aup_braces = lines(os.path.join(aup, 'examples', 'linebreak.aup'))
python_code, _ = lines(os.path.join(here, 'linebreak.py'))
ok = all(n == len(given) for n in right.values()) and len(digests) == 1
print(json.dumps({'ranges': len(starts), 'test_lines': tested, 'random_lines': 2000, 'right': right,
'paths_same': len(digests) == 1, 'aup_lines': aup_code, 'aup_brace_lines': aup_braces,
'python_lines': python_code}))
sys.exit(0 if ok else 1)
|
