From 96c35e7919e5e25187bd5adedda516a32cfc8639 Mon Sep 17 00:00:00 2001 From: eunhhu Date: Mon, 28 Sep 2026 16:27:28 +0100 Subject: Follow HarfBuzz and Unicode with one command text/upstream.py brings in a HarfBuzz release (default: the one the build links) and the UCD of the Unicode version its tables came from, builds that HarfBuzz as one static library from its amalgamated source (no system package, any version), regenerates the Unicode tables and the syllabic ones, holds the syllable machines to HarfBuzz's compiled ones, and runs the text checks built against it: Unicode's conformance tests on that UCD, random shaping against that HarfBuzz, the layout's promises, the kern class tests, HarfBuzz's own suite. --corpus adds every installed font (text/fonts.py, the corpus runner, now in the tree). What a new version breaks shows as failing checks with the font and the input to port a fix from; the generated files are left changed for review. make check-text is those checks alone; UCD and HARFBUZZ_SRC default to build/upstream's when the tool fetched them. Its first run found that text/shape-build.c used clock_gettime without asking for POSIX, building only because the system HarfBuzz's headers did. Against HarfBuzz 10.2.0 built from source and the 16.0.0 UCD, every check passes in about a minute. --- text/upstream.py | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 text/upstream.py (limited to 'text/upstream.py') diff --git a/text/upstream.py b/text/upstream.py new file mode 100644 index 0000000..ec0e18b --- /dev/null +++ b/text/upstream.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Bring in Vega's text upstreams, and hold Vega to them: + + python3 text/upstream.py [--harfbuzz VERSION] [--unicode VERSION] [--corpus] [--offline] + +Vega's shaping is HarfBuzz's, case for case, and its Unicode tables are +generated from the Unicode Character Database. When either moves on, this +is how Vega follows (docs/design/004-text.md): + + 1. fetch HarfBuzz VERSION's release (default: the one the build links, + pkg-config) and the UCD of the Unicode version its tables came from + (its src/hb-ucd-table.hh says; --unicode to choose), once, into + build/upstream; + 2. build that HarfBuzz as one static library, from its amalgamated + harfbuzz.cc: no system package, any version; + 3. regenerate text/unicode-data.c (text/unicode.py) and + text/syllabic-data.c (text/syllabic.py: the syllable machines and the + Indic, USE and Arabic tables), and hold the machines to HarfBuzz's own + compiled ones (syllabic.py --check); + 4. build Vega's text tools against that HarfBuzz (build/upstream/build) + and run make check-text: Unicode's conformance tests on that UCD, + random shaping against that HarfBuzz, the layout's promises, the kern + class tests, and HarfBuzz's own test suite, case by case; + 5. with --corpus, every installed font against it on random runs + (text/fonts.py). + +What a new version breaks shows as failing checks, each with the font and +the input (the suite's case, the random run) to port its fix from. The +generated files are left changed for review (git diff). One JSON line at +the end: the versions, what changed, how each check went; exit 1 when one +failed. --offline uses only what build/upstream already holds. +""" +import argparse +import json +import re +import subprocess +import sys +import tarfile +import urllib.request +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +UPSTREAM = ROOT / 'build' / 'upstream' + + +def fetch(url, into, offline): + if into.exists(): + return + if offline: + sys.exit(f'upstream: {into} is not here, and --offline') + print(f'fetching {url}', file=sys.stderr) + part = into.with_name(into.name + '.part') + with urllib.request.urlopen(url, timeout=300) as response, open(part, 'wb') as out: + while chunk := response.read(1 << 20): + out.write(chunk) + part.rename(into) + + +def harfbuzz(version, offline): + """The release's source, and a static library built from it.""" + source = UPSTREAM / f'harfbuzz-{version}' + if not (source / 'src' / 'harfbuzz.cc').exists(): + archive = UPSTREAM / f'harfbuzz-{version}.tar.xz' + fetch(f'https://github.com/harfbuzz/harfbuzz/releases/download/{version}/harfbuzz-{version}.tar.xz', archive, + offline) + with tarfile.open(archive) as tar: + tar.extractall(UPSTREAM, filter='data') + library = source / 'libharfbuzz.a' + if not library.exists(): + print(f'building HarfBuzz {version}', file=sys.stderr) + subprocess.run(['c++', '-O2', '-fPIC', '-std=c++17', '-DNDEBUG', '-c', str(source / 'src' / 'harfbuzz.cc'), + '-o', str(source / 'harfbuzz.o')], check=True) + subprocess.run(['ar', 'rcs', str(library), str(source / 'harfbuzz.o')], check=True) + return source, library + + +def ucd(version, offline): + directory = UPSTREAM / f'ucd-{version}' + if not (directory / 'ReadMe.txt').exists(): + archive = UPSTREAM / f'UCD-{version}.zip' + fetch(f'https://www.unicode.org/Public/{version}/ucd/UCD.zip', archive, offline) + with zipfile.ZipFile(archive) as z: + z.extractall(directory) + return directory + + +def run(name, command, results, **kwargs): + """A step: its last JSON line (if any) and whether it passed.""" + print(f'== {name}', file=sys.stderr) + done = subprocess.run(command, capture_output=True, text=True, cwd=ROOT, **kwargs) + sys.stderr.write(done.stderr[-4000:]) + reports = [json.loads(line) for line in done.stdout.splitlines() if line.startswith('{')] + results[name] = {'passed': done.returncode == 0, **({'report': reports} if reports else {})} + if done.returncode: + sys.stderr.write(done.stdout[-4000:]) + return done + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--harfbuzz') + parser.add_argument('--unicode') + parser.add_argument('--corpus', action='store_true') + parser.add_argument('--offline', action='store_true') + args = parser.parse_args() + UPSTREAM.mkdir(parents=True, exist_ok=True) + version = args.harfbuzz or subprocess.run(['pkg-config', '--modversion', 'harfbuzz'], capture_output=True, + text=True, check=True).stdout.strip() + source, library = harfbuzz(version, args.offline) + named = re.search(r'Unicode (\d+\.\d+\.\d+)', (source / 'src' / 'hb-ucd-table.hh').read_text()) + unicode = args.unicode or named.group(1) + database = ucd(unicode, args.offline) + + results = {} + generated = ['text/unicode-data.c', 'text/syllabic-data.c'] + with open(ROOT / 'text' / 'unicode-data.c', 'w') as out: + results['unicode.py'] = {'passed': subprocess.run([sys.executable, 'text/unicode.py', str(database)], + stdout=out, cwd=ROOT).returncode == 0} + with open(ROOT / 'text' / 'syllabic-data.c', 'w') as out: + results['syllabic.py'] = {'passed': subprocess.run([sys.executable, 'text/syllabic.py', str(source / 'src')], + stdout=out, cwd=ROOT).returncode == 0} + run('syllabic.py --check', [sys.executable, 'text/syllabic.py', str(source / 'src'), '--check'], results) + + build = UPSTREAM / 'build' + flags = [f'BUILD={build}', f'HARFBUZZ_CFLAGS=-I{source / "src"}', + f'HARFBUZZ_LIBS={library} -lstdc++ -lm', f'UCD={database}', f'HARFBUZZ_SRC={source}'] + run('check-text', ['make', '-s', 'check-text'] + flags, results) + if args.corpus: + run('fonts', [sys.executable, 'text/fonts.py', str(build / 'vega-symbols')], results) + + changed = subprocess.run(['git', 'diff', '--stat', '--'] + generated, capture_output=True, text=True, + cwd=ROOT).stdout.strip().splitlines() + print(json.dumps({'harfbuzz': version, 'unicode': unicode, 'changed': changed[:-1] if changed else [], + 'checks': results})) + return 0 if all(r['passed'] for r in results.values()) else 1 + + +if __name__ == '__main__': + sys.exit(main()) -- cgit v1.2.3