diff options
Diffstat (limited to 'text')
| -rw-r--r-- | text/fonts.py | 95 | ||||
| -rw-r--r-- | text/shape-build.c | 1 | ||||
| -rw-r--r-- | text/upstream.py | 140 |
3 files changed, 236 insertions, 0 deletions
diff --git a/text/fonts.py b/text/fonts.py new file mode 100644 index 0000000..1de017b --- /dev/null +++ b/text/fonts.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Every font here against HarfBuzz, on random runs: + + python3 text/fonts.py VEGA_SYMBOLS [--runs N] [--only TEXT] [FONT...] + +The fonts given, else every font fontconfig lists (of a collection, its +Korean face: CJK collections are why there are collections here). Each is +built (those with outlines) into a symbol file with its characters in the ranges below and +shaping for each script it has ten letters of (and Common); the build +refuses a file whose shaping differs from HarfBuzz's (vega-symbols build +--runs). A line per font; the last is JSON: fonts, plans, runs that +differed, fonts that failed. Needs fontTools; exits 1 on a difference. +""" +import argparse +import json +import os +import subprocess +import sys +import tempfile + +from fontTools.ttLib import TTCollection, TTFont + +RANGES = [(0x20, 0x24F), (0x250, 0x2FF), (0x300, 0x36F), (0x370, 0x52F), (0x590, 0x5FF), (0x600, 0x6FF), + (0x750, 0x77F), (0x1E00, 0x1FFF), (0x2000, 0x206F), (0x20A0, 0x20CF), (0xFB00, 0xFB4F), (0xFE70, 0xFEFF), + (0x1100, 0x11FF), (0x302E, 0x302F), (0x3130, 0x318F), (0xA960, 0xA97F), (0xAC00, 0xD7A3), (0xD7B0, 0xD7FF), + (0x900, 0xD7F), (0x1CD0, 0x1CFF), (0xA8E0, 0xA8FF), (0x25CC, 0x25CC), (0x1000, 0x109F), (0xA9E0, 0xA9FF), + (0xAA60, 0xAA7F), (0x1780, 0x17FF), (0x19E0, 0x19FF), (0xD80, 0xDFF), (0xF00, 0xFFF), (0xA980, 0xA9DF), + (0x1B00, 0x1B7F), (0x1800, 0x18AF), (0xE00, 0xEFF), (0x700, 0x74F), (0x860, 0x86F), (0xFB50, 0xFDFF), + (0xFE00, 0xFE0F)] +LETTERS = {'Latn': (0x41, 0x7A), 'Grek': (0x391, 0x3C9), 'Cyrl': (0x410, 0x44F), 'Hebr': (0x5D0, 0x5EA), + 'Arab': (0x621, 0x64A), 'Hang': (0xAC00, 0xD7A3), 'Deva': (0x915, 0x939), 'Beng': (0x995, 0x9B9), + 'Guru': (0xA15, 0xA39), 'Gujr': (0xA95, 0xAB9), 'Orya': (0xB15, 0xB39), 'Taml': (0xB95, 0xBB9), + 'Telu': (0xC15, 0xC39), 'Knda': (0xC95, 0xCB9), 'Mlym': (0xD15, 0xD39), 'Khmr': (0x1780, 0x17B3), + 'Mymr': (0x1000, 0x102A), 'Sinh': (0xD9A, 0xDC6), 'Tibt': (0xF40, 0xF6C), 'Java': (0xA98F, 0xA9B2), + 'Bali': (0x1B13, 0x1B33), 'Mong': (0x1820, 0x1878), 'Thai': (0xE01, 0xE2E), 'Laoo': (0xE81, 0xEAE), + 'Syrc': (0x710, 0x72C)} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('tool') + parser.add_argument('fonts', nargs='*') + parser.add_argument('--runs', default='500') + parser.add_argument('--only', default='') + args = parser.parse_args() + fonts = args.fonts or sorted(set(subprocess.run(['fc-list', '--format', '%{file}\n'], capture_output=True, + text=True).stdout.split())) + total = {'fonts': 0, 'plans': 0, 'differences': 0, 'failed': 0} + with tempfile.TemporaryDirectory(prefix='vega-fonts-') as work: + for path in fonts: + if args.only not in path: + continue + face = 0 + try: + if path.endswith('.ttc'): + names = [f['name'].getDebugName(4) or '' for f in TTCollection(path, lazy=True).fonts] + korean = [i for i, n in enumerate(names) if n.endswith(' KR')] + if not korean: + continue + face = korean[0] + font = TTFont(path, fontNumber=face, lazy=True) + if not any(tag in font for tag in ('glyf', 'CFF ', 'CFF2')): + continue # bitmaps alone (color emoji): nothing to draw from + cmap = font.getBestCmap() or {} + except Exception as e: # not a font fontTools reads + print('skip', path, e) + continue + codes = sorted(c for c in cmap if any(a <= c <= b for a, b in RANGES)) + if not codes: + continue + spans, start = [], codes[0] + for a, b in zip(codes, codes[1:] + [None]): + if b != a + 1: + spans.append(f'{start:#x}-{a:#x}' if a != start else f'{a:#x}') + start = b + scripts = [s for s, (a, b) in LETTERS.items() if sum(1 for c in codes if a <= c <= b) >= 10] + ['Zyyy'] + built = subprocess.run([args.tool, 'build', '--font', path] + (['--face', str(face)] if face else []) + + ['--codes', ','.join(spans), '--shape', ','.join(scripts), '--runs', args.runs, + '-o', os.path.join(work, 'font.vsym')], capture_output=True, text=True) + plans = [json.loads(line) for line in built.stdout.splitlines() if line.startswith('{"script"')] + differences = sum(p['differences'] for p in plans) + status = 'ok' if built.returncode == 0 else 'FAIL' + why = (built.stderr.strip().splitlines() or [''])[-1][:150] if built.returncode else '' + print(f'{status:4} {os.path.basename(path):40} {",".join(scripts):30} plans={len(plans):2} ' + f'differences={differences}', why, flush=True) + total['fonts'] += 1 + total['plans'] += len(plans) + total['differences'] += differences + total['failed'] += built.returncode != 0 and not differences + print(json.dumps(total)) + return 1 if total['differences'] or total['failed'] else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/text/shape-build.c b/text/shape-build.c index 1fe1c67..2a1e589 100644 --- a/text/shape-build.c +++ b/text/shape-build.c @@ -1,3 +1,4 @@ +#define _POSIX_C_SOURCE 200809L /* clock_gettime */ /* Building shaping sections with HarfBuzz (text/shape-build.h): plans as * HarfBuzz's planner compiles them (hb-ot-shape.cc's features, hb-ot-map.cc's * masks and stages), what the section carries, and a check of it against diff --git a/text/upstream.py b/text/upstream.py new file mode 100644 index 0000000..ec0e18b --- /dev/null +++ b/text/upstream.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Bring in Vega's text upstreams, and hold Vega to them: + + python3 text/upstream.py [--harfbuzz VERSION] [--unicode VERSION] [--corpus] [--offline] + +Vega's shaping is HarfBuzz's, case for case, and its Unicode tables are +generated from the Unicode Character Database. When either moves on, this +is how Vega follows (docs/design/004-text.md): + + 1. fetch HarfBuzz VERSION's release (default: the one the build links, + pkg-config) and the UCD of the Unicode version its tables came from + (its src/hb-ucd-table.hh says; --unicode to choose), once, into + build/upstream; + 2. build that HarfBuzz as one static library, from its amalgamated + harfbuzz.cc: no system package, any version; + 3. regenerate text/unicode-data.c (text/unicode.py) and + text/syllabic-data.c (text/syllabic.py: the syllable machines and the + Indic, USE and Arabic tables), and hold the machines to HarfBuzz's own + compiled ones (syllabic.py --check); + 4. build Vega's text tools against that HarfBuzz (build/upstream/build) + and run make check-text: Unicode's conformance tests on that UCD, + random shaping against that HarfBuzz, the layout's promises, the kern + class tests, and HarfBuzz's own test suite, case by case; + 5. with --corpus, every installed font against it on random runs + (text/fonts.py). + +What a new version breaks shows as failing checks, each with the font and +the input (the suite's case, the random run) to port its fix from. The +generated files are left changed for review (git diff). One JSON line at +the end: the versions, what changed, how each check went; exit 1 when one +failed. --offline uses only what build/upstream already holds. +""" +import argparse +import json +import re +import subprocess +import sys +import tarfile +import urllib.request +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +UPSTREAM = ROOT / 'build' / 'upstream' + + +def fetch(url, into, offline): + if into.exists(): + return + if offline: + sys.exit(f'upstream: {into} is not here, and --offline') + print(f'fetching {url}', file=sys.stderr) + part = into.with_name(into.name + '.part') + with urllib.request.urlopen(url, timeout=300) as response, open(part, 'wb') as out: + while chunk := response.read(1 << 20): + out.write(chunk) + part.rename(into) + + +def harfbuzz(version, offline): + """The release's source, and a static library built from it.""" + source = UPSTREAM / f'harfbuzz-{version}' + if not (source / 'src' / 'harfbuzz.cc').exists(): + archive = UPSTREAM / f'harfbuzz-{version}.tar.xz' + fetch(f'https://github.com/harfbuzz/harfbuzz/releases/download/{version}/harfbuzz-{version}.tar.xz', archive, + offline) + with tarfile.open(archive) as tar: + tar.extractall(UPSTREAM, filter='data') + library = source / 'libharfbuzz.a' + if not library.exists(): + print(f'building HarfBuzz {version}', file=sys.stderr) + subprocess.run(['c++', '-O2', '-fPIC', '-std=c++17', '-DNDEBUG', '-c', str(source / 'src' / 'harfbuzz.cc'), + '-o', str(source / 'harfbuzz.o')], check=True) + subprocess.run(['ar', 'rcs', str(library), str(source / 'harfbuzz.o')], check=True) + return source, library + + +def ucd(version, offline): + directory = UPSTREAM / f'ucd-{version}' + if not (directory / 'ReadMe.txt').exists(): + archive = UPSTREAM / f'UCD-{version}.zip' + fetch(f'https://www.unicode.org/Public/{version}/ucd/UCD.zip', archive, offline) + with zipfile.ZipFile(archive) as z: + z.extractall(directory) + return directory + + +def run(name, command, results, **kwargs): + """A step: its last JSON line (if any) and whether it passed.""" + print(f'== {name}', file=sys.stderr) + done = subprocess.run(command, capture_output=True, text=True, cwd=ROOT, **kwargs) + sys.stderr.write(done.stderr[-4000:]) + reports = [json.loads(line) for line in done.stdout.splitlines() if line.startswith('{')] + results[name] = {'passed': done.returncode == 0, **({'report': reports} if reports else {})} + if done.returncode: + sys.stderr.write(done.stdout[-4000:]) + return done + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--harfbuzz') + parser.add_argument('--unicode') + parser.add_argument('--corpus', action='store_true') + parser.add_argument('--offline', action='store_true') + args = parser.parse_args() + UPSTREAM.mkdir(parents=True, exist_ok=True) + version = args.harfbuzz or subprocess.run(['pkg-config', '--modversion', 'harfbuzz'], capture_output=True, + text=True, check=True).stdout.strip() + source, library = harfbuzz(version, args.offline) + named = re.search(r'Unicode (\d+\.\d+\.\d+)', (source / 'src' / 'hb-ucd-table.hh').read_text()) + unicode = args.unicode or named.group(1) + database = ucd(unicode, args.offline) + + results = {} + generated = ['text/unicode-data.c', 'text/syllabic-data.c'] + with open(ROOT / 'text' / 'unicode-data.c', 'w') as out: + results['unicode.py'] = {'passed': subprocess.run([sys.executable, 'text/unicode.py', str(database)], + stdout=out, cwd=ROOT).returncode == 0} + with open(ROOT / 'text' / 'syllabic-data.c', 'w') as out: + results['syllabic.py'] = {'passed': subprocess.run([sys.executable, 'text/syllabic.py', str(source / 'src')], + stdout=out, cwd=ROOT).returncode == 0} + run('syllabic.py --check', [sys.executable, 'text/syllabic.py', str(source / 'src'), '--check'], results) + + build = UPSTREAM / 'build' + flags = [f'BUILD={build}', f'HARFBUZZ_CFLAGS=-I{source / "src"}', + f'HARFBUZZ_LIBS={library} -lstdc++ -lm', f'UCD={database}', f'HARFBUZZ_SRC={source}'] + run('check-text', ['make', '-s', 'check-text'] + flags, results) + if args.corpus: + run('fonts', [sys.executable, 'text/fonts.py', str(build / 'vega-symbols')], results) + + changed = subprocess.run(['git', 'diff', '--stat', '--'] + generated, capture_output=True, text=True, + cwd=ROOT).stdout.strip().splitlines() + print(json.dumps({'harfbuzz': version, 'unicode': unicode, 'changed': changed[:-1] if changed else [], + 'checks': results})) + return 0 if all(r['passed'] for r in results.values()) else 1 + + +if __name__ == '__main__': + sys.exit(main()) |