Follow HarfBuzz and Unicode with one command

text/upstream.py brings in a HarfBuzz release (default: the one the
build links) and the UCD of the Unicode version its tables came from,
builds that HarfBuzz as one static library from its amalgamated source
(no system package, any version), regenerates the Unicode tables and the
syllabic ones, holds the syllable machines to HarfBuzz's compiled ones,
and runs the text checks built against it: Unicode's conformance tests on
that UCD, random shaping against that HarfBuzz, the layout's promises,
the kern class tests, HarfBuzz's own suite. --corpus adds every installed
font (text/fonts.py, the corpus runner, now in the tree). What a new
version breaks shows as failing checks with the font and the input to
port a fix from; the generated files are left changed for review.

make check-text is those checks alone; UCD and HARFBUZZ_SRC default to
build/upstream's when the tool fetched them. Its first run found that
text/shape-build.c used clock_gettime without asking for POSIX, building
only because the system HarfBuzz's headers did. Against HarfBuzz 10.2.0
built from source and the 16.0.0 UCD, every check passes in about a
minute.

3 files changed, 236 insertions, 0 deletions

diff --git a/text/fonts.py b/text/fonts.py
new file mode 100644
index 0000000..1de017b
--- /dev/null
+++ b/text/fonts.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Every font here against HarfBuzz, on random runs:
+
+    python3 text/fonts.py VEGA_SYMBOLS [--runs N] [--only TEXT] [FONT...]
+
+The fonts given, else every font fontconfig lists (of a collection, its
+Korean face: CJK collections are why there are collections here). Each is
+built (those with outlines) into a symbol file with its characters in the ranges below and
+shaping for each script it has ten letters of (and Common); the build
+refuses a file whose shaping differs from HarfBuzz's (vega-symbols build
+--runs). A line per font; the last is JSON: fonts, plans, runs that
+differed, fonts that failed. Needs fontTools; exits 1 on a difference.
+"""
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+from fontTools.ttLib import TTCollection, TTFont
+
+RANGES = [(0x20, 0x24F), (0x250, 0x2FF), (0x300, 0x36F), (0x370, 0x52F), (0x590, 0x5FF), (0x600, 0x6FF),
+          (0x750, 0x77F), (0x1E00, 0x1FFF), (0x2000, 0x206F), (0x20A0, 0x20CF), (0xFB00, 0xFB4F), (0xFE70, 0xFEFF),
+          (0x1100, 0x11FF), (0x302E, 0x302F), (0x3130, 0x318F), (0xA960, 0xA97F), (0xAC00, 0xD7A3), (0xD7B0, 0xD7FF),
+          (0x900, 0xD7F), (0x1CD0, 0x1CFF), (0xA8E0, 0xA8FF), (0x25CC, 0x25CC), (0x1000, 0x109F), (0xA9E0, 0xA9FF),
+          (0xAA60, 0xAA7F), (0x1780, 0x17FF), (0x19E0, 0x19FF), (0xD80, 0xDFF), (0xF00, 0xFFF), (0xA980, 0xA9DF),
+          (0x1B00, 0x1B7F), (0x1800, 0x18AF), (0xE00, 0xEFF), (0x700, 0x74F), (0x860, 0x86F), (0xFB50, 0xFDFF),
+          (0xFE00, 0xFE0F)]
+LETTERS = {'Latn': (0x41, 0x7A), 'Grek': (0x391, 0x3C9), 'Cyrl': (0x410, 0x44F), 'Hebr': (0x5D0, 0x5EA),
+           'Arab': (0x621, 0x64A), 'Hang': (0xAC00, 0xD7A3), 'Deva': (0x915, 0x939), 'Beng': (0x995, 0x9B9),
+           'Guru': (0xA15, 0xA39), 'Gujr': (0xA95, 0xAB9), 'Orya': (0xB15, 0xB39), 'Taml': (0xB95, 0xBB9),
+           'Telu': (0xC15, 0xC39), 'Knda': (0xC95, 0xCB9), 'Mlym': (0xD15, 0xD39), 'Khmr': (0x1780, 0x17B3),
+           'Mymr': (0x1000, 0x102A), 'Sinh': (0xD9A, 0xDC6), 'Tibt': (0xF40, 0xF6C), 'Java': (0xA98F, 0xA9B2),
+           'Bali': (0x1B13, 0x1B33), 'Mong': (0x1820, 0x1878), 'Thai': (0xE01, 0xE2E), 'Laoo': (0xE81, 0xEAE),
+           'Syrc': (0x710, 0x72C)}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('tool')
+    parser.add_argument('fonts', nargs='*')
+    parser.add_argument('--runs', default='500')
+    parser.add_argument('--only', default='')
+    args = parser.parse_args()
+    fonts = args.fonts or sorted(set(subprocess.run(['fc-list', '--format', '%{file}\n'], capture_output=True,
+                                                    text=True).stdout.split()))
+    total = {'fonts': 0, 'plans': 0, 'differences': 0, 'failed': 0}
+    with tempfile.TemporaryDirectory(prefix='vega-fonts-') as work:
+        for path in fonts:
+            if args.only not in path:
+                continue
+            face = 0
+            try:
+                if path.endswith('.ttc'):
+                    names = [f['name'].getDebugName(4) or '' for f in TTCollection(path, lazy=True).fonts]
+                    korean = [i for i, n in enumerate(names) if n.endswith(' KR')]
+                    if not korean:
+                        continue
+                    face = korean[0]
+                font = TTFont(path, fontNumber=face, lazy=True)
+                if not any(tag in font for tag in ('glyf', 'CFF ', 'CFF2')):
+                    continue  # bitmaps alone (color emoji): nothing to draw from
+                cmap = font.getBestCmap() or {}
+            except Exception as e:  # not a font fontTools reads
+                print('skip', path, e)
+                continue
+            codes = sorted(c for c in cmap if any(a <= c <= b for a, b in RANGES))
+            if not codes:
+                continue
+            spans, start = [], codes[0]
+            for a, b in zip(codes, codes[1:] + [None]):
+                if b != a + 1:
+                    spans.append(f'{start:#x}-{a:#x}' if a != start else f'{a:#x}')
+                    start = b
+            scripts = [s for s, (a, b) in LETTERS.items() if sum(1 for c in codes if a <= c <= b) >= 10] + ['Zyyy']
+            built = subprocess.run([args.tool, 'build', '--font', path] + (['--face', str(face)] if face else []) +
+                                   ['--codes', ','.join(spans), '--shape', ','.join(scripts), '--runs', args.runs,
+                                    '-o', os.path.join(work, 'font.vsym')], capture_output=True, text=True)
+            plans = [json.loads(line) for line in built.stdout.splitlines() if line.startswith('{"script"')]
+            differences = sum(p['differences'] for p in plans)
+            status = 'ok' if built.returncode == 0 else 'FAIL'
+            why = (built.stderr.strip().splitlines() or [''])[-1][:150] if built.returncode else ''
+            print(f'{status:4} {os.path.basename(path):40} {",".join(scripts):30} plans={len(plans):2} '
+                  f'differences={differences}', why, flush=True)
+            total['fonts'] += 1
+            total['plans'] += len(plans)
+            total['differences'] += differences
+            total['failed'] += built.returncode != 0 and not differences
+    print(json.dumps(total))
+    return 1 if total['differences'] or total['failed'] else 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/text/shape-build.c b/text/shape-build.c
index 1fe1c67..2a1e589 100644
--- a/text/shape-build.c
+++ b/text/shape-build.c
@@ -1,3 +1,4 @@
+#define _POSIX_C_SOURCE 200809L /* clock_gettime */
 /* Building shaping sections with HarfBuzz (text/shape-build.h): plans as
  * HarfBuzz's planner compiles them (hb-ot-shape.cc's features, hb-ot-map.cc's
  * masks and stages), what the section carries, and a check of it against
diff --git a/text/upstream.py b/text/upstream.py
new file mode 100644
index 0000000..ec0e18b
--- /dev/null
+++ b/text/upstream.py
@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Bring in Vega's text upstreams, and hold Vega to them:
+
+    python3 text/upstream.py [--harfbuzz VERSION] [--unicode VERSION] [--corpus] [--offline]
+
+Vega's shaping is HarfBuzz's, case for case, and its Unicode tables are
+generated from the Unicode Character Database. When either moves on, this
+is how Vega follows (docs/design/004-text.md):
+
+  1. fetch HarfBuzz VERSION's release (default: the one the build links,
+     pkg-config) and the UCD of the Unicode version its tables came from
+     (its src/hb-ucd-table.hh says; --unicode to choose), once, into
+     build/upstream;
+  2. build that HarfBuzz as one static library, from its amalgamated
+     harfbuzz.cc: no system package, any version;
+  3. regenerate text/unicode-data.c (text/unicode.py) and
+     text/syllabic-data.c (text/syllabic.py: the syllable machines and the
+     Indic, USE and Arabic tables), and hold the machines to HarfBuzz's own
+     compiled ones (syllabic.py --check);
+  4. build Vega's text tools against that HarfBuzz (build/upstream/build)
+     and run make check-text: Unicode's conformance tests on that UCD,
+     random shaping against that HarfBuzz, the layout's promises, the kern
+     class tests, and HarfBuzz's own test suite, case by case;
+  5. with --corpus, every installed font against it on random runs
+     (text/fonts.py).
+
+What a new version breaks shows as failing checks, each with the font and
+the input (the suite's case, the random run) to port its fix from. The
+generated files are left changed for review (git diff). One JSON line at
+the end: the versions, what changed, how each check went; exit 1 when one
+failed. --offline uses only what build/upstream already holds.
+"""
+import argparse
+import json
+import re
+import subprocess
+import sys
+import tarfile
+import urllib.request
+import zipfile
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+UPSTREAM = ROOT / 'build' / 'upstream'
+
+
+def fetch(url, into, offline):
+    if into.exists():
+        return
+    if offline:
+        sys.exit(f'upstream: {into} is not here, and --offline')
+    print(f'fetching {url}', file=sys.stderr)
+    part = into.with_name(into.name + '.part')
+    with urllib.request.urlopen(url, timeout=300) as response, open(part, 'wb') as out:
+        while chunk := response.read(1 << 20):
+            out.write(chunk)
+    part.rename(into)
+
+
+def harfbuzz(version, offline):
+    """The release's source, and a static library built from it."""
+    source = UPSTREAM / f'harfbuzz-{version}'
+    if not (source / 'src' / 'harfbuzz.cc').exists():
+        archive = UPSTREAM / f'harfbuzz-{version}.tar.xz'
+        fetch(f'https://github.com/harfbuzz/harfbuzz/releases/download/{version}/harfbuzz-{version}.tar.xz', archive,
+              offline)
+        with tarfile.open(archive) as tar:
+            tar.extractall(UPSTREAM, filter='data')
+    library = source / 'libharfbuzz.a'
+    if not library.exists():
+        print(f'building HarfBuzz {version}', file=sys.stderr)
+        subprocess.run(['c++', '-O2', '-fPIC', '-std=c++17', '-DNDEBUG', '-c', str(source / 'src' / 'harfbuzz.cc'),
+                        '-o', str(source / 'harfbuzz.o')], check=True)
+        subprocess.run(['ar', 'rcs', str(library), str(source / 'harfbuzz.o')], check=True)
+    return source, library
+
+
+def ucd(version, offline):
+    directory = UPSTREAM / f'ucd-{version}'
+    if not (directory / 'ReadMe.txt').exists():
+        archive = UPSTREAM / f'UCD-{version}.zip'
+        fetch(f'https://www.unicode.org/Public/{version}/ucd/UCD.zip', archive, offline)
+        with zipfile.ZipFile(archive) as z:
+            z.extractall(directory)
+    return directory
+
+
+def run(name, command, results, **kwargs):
+    """A step: its last JSON line (if any) and whether it passed."""
+    print(f'== {name}', file=sys.stderr)
+    done = subprocess.run(command, capture_output=True, text=True, cwd=ROOT, **kwargs)
+    sys.stderr.write(done.stderr[-4000:])
+    reports = [json.loads(line) for line in done.stdout.splitlines() if line.startswith('{')]
+    results[name] = {'passed': done.returncode == 0, **({'report': reports} if reports else {})}
+    if done.returncode:
+        sys.stderr.write(done.stdout[-4000:])
+    return done
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--harfbuzz')
+    parser.add_argument('--unicode')
+    parser.add_argument('--corpus', action='store_true')
+    parser.add_argument('--offline', action='store_true')
+    args = parser.parse_args()
+    UPSTREAM.mkdir(parents=True, exist_ok=True)
+    version = args.harfbuzz or subprocess.run(['pkg-config', '--modversion', 'harfbuzz'], capture_output=True,
+                                              text=True, check=True).stdout.strip()
+    source, library = harfbuzz(version, args.offline)
+    named = re.search(r'Unicode (\d+\.\d+\.\d+)', (source / 'src' / 'hb-ucd-table.hh').read_text())
+    unicode = args.unicode or named.group(1)
+    database = ucd(unicode, args.offline)
+
+    results = {}
+    generated = ['text/unicode-data.c', 'text/syllabic-data.c']
+    with open(ROOT / 'text' / 'unicode-data.c', 'w') as out:
+        results['unicode.py'] = {'passed': subprocess.run([sys.executable, 'text/unicode.py', str(database)],
+                                                          stdout=out, cwd=ROOT).returncode == 0}
+    with open(ROOT / 'text' / 'syllabic-data.c', 'w') as out:
+        results['syllabic.py'] = {'passed': subprocess.run([sys.executable, 'text/syllabic.py', str(source / 'src')],
+                                                           stdout=out, cwd=ROOT).returncode == 0}
+    run('syllabic.py --check', [sys.executable, 'text/syllabic.py', str(source / 'src'), '--check'], results)
+
+    build = UPSTREAM / 'build'
+    flags = [f'BUILD={build}', f'HARFBUZZ_CFLAGS=-I{source / "src"}',
+             f'HARFBUZZ_LIBS={library} -lstdc++ -lm', f'UCD={database}', f'HARFBUZZ_SRC={source}']
+    run('check-text', ['make', '-s', 'check-text'] + flags, results)
+    if args.corpus:
+        run('fonts', [sys.executable, 'text/fonts.py', str(build / 'vega-symbols')], results)
+
+    changed = subprocess.run(['git', 'diff', '--stat', '--'] + generated, capture_output=True, text=True,
+                             cwd=ROOT).stdout.strip().splitlines()
+    print(json.dumps({'harfbuzz': version, 'unicode': unicode, 'changed': changed[:-1] if changed else [],
+                      'checks': results}))
+    return 0 if all(r['passed'] for r in results.values()) else 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())