diff options
Diffstat (limited to 'research/experiments/044-specs/linebreak_check.py')
| -rw-r--r-- | research/experiments/044-specs/linebreak_check.py | 147 |
1 files changed, 147 insertions, 0 deletions
diff --git a/research/experiments/044-specs/linebreak_check.py b/research/experiments/044-specs/linebreak_check.py new file mode 100644 index 0000000..3de4ccf --- /dev/null +++ b/research/experiments/044-specs/linebreak_check.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Research 140: line breaking (UAX #14, Unicode 16.0) written in aup (aup's examples/linebreak.aup). + +Builds the properties' table from the UCD Vega's text is frozen at (its text/unicode.py reads the derived files): ranges +of code points, each its class after LB1, initial or final quotation (Pi, Pf), East Asian (F, W, H), and pictographic and +unassigned (Extended_Pictographic and Cn). Loads it into the app (a load message's rows), then gives it every line of +LineBreakTest.txt and random lines (code points drawn from every class), a line a message, on Pure's three paths +(batches, rounds, all compiled): each line's boundaries must be the test's (the random lines': linebreak.py's, which +passes the test whole), the paths the same. pure-run prints views alone (--views): the table would be in every line. +Then counts lines (code lines: neither blank nor a comment; brace lines: a closing brace alone) of linebreak.aup and of +linebreak.py. One JSON line; given a directory, it writes the table and the lines there for linebreak_time.c. + + python3 linebreak_check.py [DIRECTORY] [LINES] (LINES: the test's lines to take, all by default)""" +import json, os, random, struct, subprocess, sys, tempfile +from pathlib import Path + +here = os.path.dirname(os.path.abspath(__file__)) +pure = os.path.normpath(os.path.join(here, '../../..')) +aup = os.environ.get('AUP', os.path.join(pure, '..', 'aup')) +vega = os.environ.get('VEGA', os.path.join(pure, '..', 'vega')) +ucd_path = Path(vega) / 'build/upstream/ucd-16.0.0' +sys.path.insert(0, here) +import linebreak # noqa: E402 + +CLASSES = 'BK CR LF NL SP ZW ZWJ CM WJ GL BA BB B2 HY CB CL CP EX IN NS OP QU IS NU PO PR SY AK AL AP AS EB EM H2 H3 HL ID JL JV JT RI VF VI'.split() + + +def lines(path, stop=None): + code = braces = 0 + for line in open(path): + if stop and line.startswith(stop): break + text = line.strip() + if not text or text.startswith('#'): continue + code += 1 + braces += text == '}' + return code, braces + + +def table(): + """The ranges' starts and their rows (class, pi, pf, east, pictographic), every code point in one.""" + argv, sys.argv = sys.argv, ['unicode.py', str(ucd_path)] + sys.path.insert(0, os.path.join(vega, 'text')) + import unicode as ucd + sys.argv = argv + line = ucd.derived('extracted/DerivedLineBreak.txt', ucd.LINE, ucd.aliases('lb')) + category = ucd.derived('extracted/DerivedGeneralCategory.txt', ucd.CATEGORIES) + width = ucd.derived('extracted/DerivedEastAsianWidth.txt', ucd.WIDTHS, ucd.aliases('ea')) + pictographic = bytearray(ucd.LAST + 1) + for first, last, fields, missing in ucd.ranges('emoji/emoji-data.txt'): + if not missing and fields[0] == 'Extended_Pictographic': + pictographic[first:last + 1] = b'\1' * (last - first + 1) + starts, rows = [], [] + for c in range(ucd.LAST + 1): + name, g = ucd.LINE[line[c]], ucd.CATEGORIES[category[c]] + if name in ('AI', 'SG', 'XX'): name = 'AL' # LB1 + elif name == 'SA': name = 'CM' if g in ('Mn', 'Mc') else 'AL' + elif name == 'CJ': name = 'NS' + row = (CLASSES.index(name), g == 'Pi', g == 'Pf', ucd.WIDTHS[width[c]] in ('F', 'W', 'H'), bool(pictographic[c]) and g == 'Cn') + if not rows or rows[-1] != row: + starts.append(c) + rows.append(row) + return starts, rows + + +def test_lines(limit=None): + out = [] + for text in open(ucd_path / 'auxiliary/LineBreakTest.txt', encoding='utf-8'): + body = text.split('#', 1)[0].split() + if body: + out.append(([int(x, 16) for x in body if x not in ('÷', '×')], [x == '÷' for x in body if x in ('÷', '×')])) + return out[:limit] if limit else out + + +def random_lines(chooser, starts, rows, count): + """Lines of code points drawn from each class's ranges (and from the specials the rules name), their boundaries + as linebreak.py works them out.""" + by_class = {} + for k, (start, row) in enumerate(zip(starts, rows)): + end = starts[k + 1] if k + 1 < len(starts) else 0x110000 + by_class.setdefault(row[0], []).append((start, end)) + specials = [0x2010, 0x25CC, 0x0020, 0x000A, 0x000D, 0x200D, 0x00AB, 0x00BB, 0x201C, 0x201D, 0x1F1E6, 0x1F3FB, 0x1F466] + out = [] + for _ in range(count): + codes = [] + for _ in range(chooser.randrange(1, 9)): + if chooser.random() < 0.15: + codes.append(chooser.choice(specials)) + else: + start, end = chooser.choice(by_class[chooser.choice(list(by_class))]) + codes.append(chooser.randrange(start, end)) + out.append((codes, [b != linebreak.NONE for b in linebreak.breaks(codes, starts, rows)])) + return out + + +def run(module, load, given, env): + """Each line's boundaries as the app gives them (None where its event failed), pure-run's output streamed.""" + commands = load + '.commands' + with open(commands, 'w') as f: + f.write(f'message load @{load}\n') + for codes, _ in given: + f.write(f"message line {b''.join(struct.pack('<d', c) for c in codes).hex() or '-'}\n") + with open(commands) as stdin: + process = subprocess.Popen([os.path.join(pure, 'build', 'pure-run'), module, '--views'], stdin=stdin, + stdout=subprocess.PIPE, text=True, env=dict(os.environ, **env)) + got, digest = [], 0 + for k, line in enumerate(process.stdout): + digest = hash((digest, line)) + if k == 0: continue # the load + state = json.loads(line) + got.append([b != 0 for b in bytes.fromhex(state['views']['boundaries'])] if state['fault'] == 'none' else None) + process.wait() + return got, digest + + +if __name__ == '__main__': + starts, rows = table() + for codes, want in test_lines(): # the reference passes the test whole first + assert [b != linebreak.NONE for b in linebreak.breaks(codes, starts, rows)] == want, codes + given = test_lines(int(sys.argv[2]) if len(sys.argv) > 2 else None) + given += random_lines(random.Random(140), starts, rows, 2000) + with tempfile.TemporaryDirectory() as work: + load = os.path.join(work, 'ranges.bin') + with open(load, 'wb') as f: + f.write(b''.join(struct.pack('<6d', start, *row) for start, row in zip(starts, rows))) + if len(sys.argv) > 1 and sys.argv[1]: + os.makedirs(sys.argv[1], exist_ok=True) + with open(os.path.join(sys.argv[1], 'ranges.bin'), 'wb') as f, open(load, 'rb') as g: + f.write(g.read()) + with open(os.path.join(sys.argv[1], 'lines.bin'), 'wb') as f: # each: u32 count, its code points, count + 1 bytes + for codes, want in given: + f.write(struct.pack('<I', len(codes)) + struct.pack(f'<{len(codes)}I', *codes) + bytes(want)) + module = os.path.join(work, 'linebreak.pmod') + subprocess.run([os.path.join(aup, 'build', 'aup'), 'build', os.path.join(aup, 'examples', 'linebreak.aup'), '-o', module], + check=True) + right, digests = {}, set() + for name, env in (('batches', {'PURE_JIT': '0'}), ('rounds', {'PURE_JIT': '0', 'PURE_BATCH': '0'}), ('all', {'PURE_JIT': 'all'})): + got, digest = run(module, load, given, env) + right[name] = sum(g == want for g, (_, want) in zip(got, given)) if len(got) == len(given) else 0 + digests.add(digest) + tested = len(given) - 2000 + aup_code, aup_braces = lines(os.path.join(aup, 'examples', 'linebreak.aup')) + python_code, _ = lines(os.path.join(here, 'linebreak.py')) + ok = all(n == len(given) for n in right.values()) and len(digests) == 1 + print(json.dumps({'ranges': len(starts), 'test_lines': tested, 'random_lines': 2000, 'right': right, + 'paths_same': len(digests) == 1, 'aup_lines': aup_code, 'aup_brace_lines': aup_braces, + 'python_lines': python_code})) + sys.exit(0 if ok else 1) |