diff options
| -rw-r--r-- | Makefile | 28 | ||||
| -rw-r--r-- | docs/design/004-text.md | 28 | ||||
| -rw-r--r-- | text/fonts.py | 95 | ||||
| -rw-r--r-- | text/shape-build.c | 1 | ||||
| -rw-r--r-- | text/upstream.py | 140 |
5 files changed, 281 insertions, 11 deletions
@@ -15,7 +15,7 @@ LIBPURE = $(PURE)/build/libpure.a SURFACES = surfaces/screen.c surfaces/draw.c surfaces/width.c surfaces/web.c surfaces/pixels.c surfaces/display.c \ surfaces/input.c surfaces/hangul.c surfaces/dbus.c surfaces/access.c surfaces/main.c -.PHONY: all check check-harfbuzz FORCE +.PHONY: all check check-text check-harfbuzz FORCE all: $(BUILD)/vega $(BUILD)/vega-symbols $(BUILD)/vega-surface.so $(BUILD): @@ -69,29 +69,35 @@ $(BUILD)/test-layout: text/test-layout.c $(LAYOUT) text/layout.h text/shape.h te symbols/symbols.h | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) -Itext -Isymbols text/test-layout.c $(LAYOUT) symbols/symbols.c $(LDFLAGS) -lm $(LDLIBS) -o $@ -# Unicode's conformance tests, where the UCD is installed (Debian: unicode-data); -# and shaping, where DejaVu Sans is: the build refuses a file whose shaping -# differs from HarfBuzz's on random runs of Latin, Greek, Cyrillic, Hebrew, -# Arabic and Common text, both ways. -UCD ?= /usr/share/unicode +# Unicode's conformance tests, on the UCD the tables were made from: the one +# text/upstream.py fetches into build/upstream, else the installed one +# (Debian: unicode-data), which test-text passes over when its version is +# not the tables'. And shaping, where DejaVu Sans is: the build refuses a +# file whose shaping differs from HarfBuzz's on random runs of Latin, Greek, +# Cyrillic, Hebrew, Arabic and Common text, both ways. +UNICODE_VERSION := $(shell sed -n '1s/.*Database \([0-9.]*\):.*/\1/p' text/unicode-data.c) +UCD ?= $(if $(wildcard build/upstream/ucd-$(UNICODE_VERSION)/ReadMe.txt),build/upstream/ucd-$(UNICODE_VERSION),/usr/share/unicode) SHAPED_FONT ?= /usr/share/fonts/truetype/dejavu/DejaVuSans.ttf SHAPED_CODES = 0x20-0x7e,0xa0-0x24f,0x300-0x36f,0x370-0x3ff,0x400-0x4ff,0x591-0x5f4,0x600-0x6ff,0x2000-0x206f # HarfBuzz's own shaping tests, where a HarfBuzz release's source is -# (HARFBUZZ_SRC: the same version the build links): each case shaped by -# Vega and by HarfBuzz must agree, but for those text/harfbuzz-suite.known -# lists. -HARFBUZZ_SRC ?= +# (HARFBUZZ_SRC: the same version the build links; build/upstream's, when +# text/upstream.py fetched one): each case shaped by Vega and by HarfBuzz +# must agree, but for those text/harfbuzz-suite.known lists. +HARFBUZZ_SRC ?= $(lastword $(wildcard build/upstream/harfbuzz-*)) check-harfbuzz: $(BUILD)/vega-symbols if [ -d "$(HARFBUZZ_SRC)/test/shape/data" ]; then python3 text/harfbuzz-suite.py $(HARFBUZZ_SRC) $(BUILD)/vega-symbols; \ else echo "no HarfBuzz source at HARFBUZZ_SRC: its test suite skipped"; fi -check: $(BUILD)/vega $(BUILD)/vega-symbols $(BUILD)/vega-surface.so $(BUILD)/test-text $(BUILD)/test-layout check-harfbuzz +# What text/upstream.py runs against a new HarfBuzz and UCD: the text alone. +check-text: $(BUILD)/vega-symbols $(BUILD)/test-text $(BUILD)/test-layout check-harfbuzz if [ -f $(UCD)/BidiTest.txt ]; then $(BUILD)/test-text $(UCD); else echo "no UCD at $(UCD): text tests skipped"; fi if [ -f $(SHAPED_FONT) ]; then $(BUILD)/vega-symbols build --font $(SHAPED_FONT) --codes $(SHAPED_CODES) \ --shape Latn,Grek,Cyrl,Hebr,Arab,Zyyy --runs 5000 -o $(BUILD)/shaped.vsym && \ $(BUILD)/test-layout $(BUILD)/shaped.vsym; else echo "no $(SHAPED_FONT): shaping tests skipped"; fi if [ -f $(SHAPED_FONT) ] && python3 -c 'import fontTools' 2>/dev/null; then \ python3 text/test-kern.py $(BUILD)/vega-symbols $(SHAPED_FONT); else echo "no fontTools: kern class tests skipped"; fi + +check: $(BUILD)/vega $(BUILD)/vega-surface.so check-text $(MAKE) -C $(AUP) build/aup $(MAKE) -C $(PURE) build/pure-serve python3 tests/check.py --vega $(BUILD)/vega --symbols $(BUILD)/vega-symbols --aup $(AUP)/build/aup --serve $(PURE)/build/pure-serve diff --git a/docs/design/004-text.md b/docs/design/004-text.md index b9727ae..502eeb2 100644 --- a/docs/design/004-text.md +++ b/docs/design/004-text.md @@ -267,6 +267,34 @@ GSUB가 끝없이 늘리는 글꼴(text-rendering-tests의 GSUB-3)에서 Vega는 마지막으로 끝난 패스의 글리프로 계속한다. 그 뒤의 패스는 아무것도 하지 않고, 위치는 기본 advance가 된다. Vega도 그렇게 한다. 깨진 글꼴에서도 글이 사라지지 않고 HarfBuzz와 같은 모습으로 남는다(5,018건이 같다). +## 윗물을 따라가기: HarfBuzz와 유니코드가 올라갈 때 + +Vega의 셰이핑은 HarfBuzz와 경우마다 같아야 하고, 유니코드 성질 표는 UCD에서 만든다. 둘 다 해마다 움직인다. 유니코드는 +새 문자와 문자 체계, 바뀐 규칙을 들이고, HarfBuzz는 새 문자 체계를 셰이퍼에 넣고 글꼴마다의 버릇을 고친다. 따라가는 일은 +손으로 옮기는 것과 기계로 다시 만드는 것으로 나뉜다. 기계로 할 수 있는 것은 모두 기계가 한다. + +- 다시 만드는 것(데이터): 성질 표는 UCD에서([text/unicode.py](../../text/unicode.py)), 음절 기계와 인도계·USE·아랍 표는 + HarfBuzz의 원본에서([text/syllabic.py](../../text/syllabic.py)) 만든다. 기계는 HarfBuzz가 컴파일한 것과 대 본다. +- 옮기는 것(논리): 셰이퍼와 규칙의 코드. 무엇을 옮길지는 검사가 가리킨다. 실패마다 글꼴과 입력(시험 모음의 경우, + 무작위 실행)이 붙어 나온다. +- 검사: 유니코드의 적합성 시험, 무작위 실행으로 HarfBuzz와 대기, HarfBuzz의 시험 모음(글꼴 버릇의 목록), 설치된 + 글꼴 전부. 빌드 도구는 HarfBuzz와 다르게 셰이핑하는 심볼 파일을 만들지 않는다. 그래서 옮기지 못한 버릇은 틀린 글이 아니라 + 거부된 글꼴로 드러난다. + +[text/upstream.py](../../text/upstream.py)가 이 일을 한 명령으로 한다. `python3 text/upstream.py --harfbuzz 버전`은 +그 릴리스의 원본과, 그 HarfBuzz의 표가 기준으로 삼은 유니코드 판의 UCD를 `build/upstream`에 받는다. 그 HarfBuzz를 +하나로 묶인 원본(`harfbuzz.cc`)에서 정적 라이브러리로 빌드한다. 시스템 꾸러미가 필요 없고 어느 판이든 된다. 두 생성 파일을 +다시 만들고 기계를 대 본다. 그 HarfBuzz에 링크한 도구로 `make check-text`(적합성 시험, 무작위 셰이핑, 글줄의 약속, kern 등급 +시험, HarfBuzz 시험 모음)를 돌린다. `--corpus`면 설치된 글꼴 전부를 대 본다([text/fonts.py](../../text/fonts.py)). +생성 파일은 바뀐 채로 두어 검토하게 하고, 판과 바뀐 것과 검사마다의 결과를 JSON 한 줄로 낸다. HarfBuzz 10.2.0과 유니코드 +16.0.0에서 받기를 빼고 1분 남짓 걸린다(원본에서 빌드한 HarfBuzz로 모든 검사가 통과한다). 처음 돌렸을 때 하나를 찾았다. +`shape-build.c`가 `clock_gettime`을 쓰면서 POSIX를 스스로 선언하지 않아, 시스템 HarfBuzz의 머리 파일을 거칠 때만 빌드되고 +있었다. + +한계도 있다. 무작위 대기와 시험 모음은 증명이 아니다. 드문 입력의 차이는 남을 수 있다. 셰이퍼 논리는 여전히 판마다 손으로 +옮겨야 한다. 도구가 줄여 주는 것은 무엇을 옮길지 찾는 일이다. AAT와 사용자 기능, 가변 글꼴은 아직 받지 않는다(시험 모음에서 +빼고 센다). + ## 비례폭 글줄 [text/layout.c](../../text/layout.c)는 검증한 부품을 UAX #9 3.4절과 UAX #14가 말하는 순서로 잇는다. diff --git a/text/fonts.py b/text/fonts.py new file mode 100644 index 0000000..1de017b --- /dev/null +++ b/text/fonts.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Every font here against HarfBuzz, on random runs: + + python3 text/fonts.py VEGA_SYMBOLS [--runs N] [--only TEXT] [FONT...] + +The fonts given, else every font fontconfig lists (of a collection, its +Korean face: CJK collections are why there are collections here). Each is +built (those with outlines) into a symbol file with its characters in the ranges below and +shaping for each script it has ten letters of (and Common); the build +refuses a file whose shaping differs from HarfBuzz's (vega-symbols build +--runs). A line per font; the last is JSON: fonts, plans, runs that +differed, fonts that failed. Needs fontTools; exits 1 on a difference. +""" +import argparse +import json +import os +import subprocess +import sys +import tempfile + +from fontTools.ttLib import TTCollection, TTFont + +RANGES = [(0x20, 0x24F), (0x250, 0x2FF), (0x300, 0x36F), (0x370, 0x52F), (0x590, 0x5FF), (0x600, 0x6FF), + (0x750, 0x77F), (0x1E00, 0x1FFF), (0x2000, 0x206F), (0x20A0, 0x20CF), (0xFB00, 0xFB4F), (0xFE70, 0xFEFF), + (0x1100, 0x11FF), (0x302E, 0x302F), (0x3130, 0x318F), (0xA960, 0xA97F), (0xAC00, 0xD7A3), (0xD7B0, 0xD7FF), + (0x900, 0xD7F), (0x1CD0, 0x1CFF), (0xA8E0, 0xA8FF), (0x25CC, 0x25CC), (0x1000, 0x109F), (0xA9E0, 0xA9FF), + (0xAA60, 0xAA7F), (0x1780, 0x17FF), (0x19E0, 0x19FF), (0xD80, 0xDFF), (0xF00, 0xFFF), (0xA980, 0xA9DF), + (0x1B00, 0x1B7F), (0x1800, 0x18AF), (0xE00, 0xEFF), (0x700, 0x74F), (0x860, 0x86F), (0xFB50, 0xFDFF), + (0xFE00, 0xFE0F)] +LETTERS = {'Latn': (0x41, 0x7A), 'Grek': (0x391, 0x3C9), 'Cyrl': (0x410, 0x44F), 'Hebr': (0x5D0, 0x5EA), + 'Arab': (0x621, 0x64A), 'Hang': (0xAC00, 0xD7A3), 'Deva': (0x915, 0x939), 'Beng': (0x995, 0x9B9), + 'Guru': (0xA15, 0xA39), 'Gujr': (0xA95, 0xAB9), 'Orya': (0xB15, 0xB39), 'Taml': (0xB95, 0xBB9), + 'Telu': (0xC15, 0xC39), 'Knda': (0xC95, 0xCB9), 'Mlym': (0xD15, 0xD39), 'Khmr': (0x1780, 0x17B3), + 'Mymr': (0x1000, 0x102A), 'Sinh': (0xD9A, 0xDC6), 'Tibt': (0xF40, 0xF6C), 'Java': (0xA98F, 0xA9B2), + 'Bali': (0x1B13, 0x1B33), 'Mong': (0x1820, 0x1878), 'Thai': (0xE01, 0xE2E), 'Laoo': (0xE81, 0xEAE), + 'Syrc': (0x710, 0x72C)} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('tool') + parser.add_argument('fonts', nargs='*') + parser.add_argument('--runs', default='500') + parser.add_argument('--only', default='') + args = parser.parse_args() + fonts = args.fonts or sorted(set(subprocess.run(['fc-list', '--format', '%{file}\n'], capture_output=True, + text=True).stdout.split())) + total = {'fonts': 0, 'plans': 0, 'differences': 0, 'failed': 0} + with tempfile.TemporaryDirectory(prefix='vega-fonts-') as work: + for path in fonts: + if args.only not in path: + continue + face = 0 + try: + if path.endswith('.ttc'): + names = [f['name'].getDebugName(4) or '' for f in TTCollection(path, lazy=True).fonts] + korean = [i for i, n in enumerate(names) if n.endswith(' KR')] + if not korean: + continue + face = korean[0] + font = TTFont(path, fontNumber=face, lazy=True) + if not any(tag in font for tag in ('glyf', 'CFF ', 'CFF2')): + continue # bitmaps alone (color emoji): nothing to draw from + cmap = font.getBestCmap() or {} + except Exception as e: # not a font fontTools reads + print('skip', path, e) + continue + codes = sorted(c for c in cmap if any(a <= c <= b for a, b in RANGES)) + if not codes: + continue + spans, start = [], codes[0] + for a, b in zip(codes, codes[1:] + [None]): + if b != a + 1: + spans.append(f'{start:#x}-{a:#x}' if a != start else f'{a:#x}') + start = b + scripts = [s for s, (a, b) in LETTERS.items() if sum(1 for c in codes if a <= c <= b) >= 10] + ['Zyyy'] + built = subprocess.run([args.tool, 'build', '--font', path] + (['--face', str(face)] if face else []) + + ['--codes', ','.join(spans), '--shape', ','.join(scripts), '--runs', args.runs, + '-o', os.path.join(work, 'font.vsym')], capture_output=True, text=True) + plans = [json.loads(line) for line in built.stdout.splitlines() if line.startswith('{"script"')] + differences = sum(p['differences'] for p in plans) + status = 'ok' if built.returncode == 0 else 'FAIL' + why = (built.stderr.strip().splitlines() or [''])[-1][:150] if built.returncode else '' + print(f'{status:4} {os.path.basename(path):40} {",".join(scripts):30} plans={len(plans):2} ' + f'differences={differences}', why, flush=True) + total['fonts'] += 1 + total['plans'] += len(plans) + total['differences'] += differences + total['failed'] += built.returncode != 0 and not differences + print(json.dumps(total)) + return 1 if total['differences'] or total['failed'] else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/text/shape-build.c b/text/shape-build.c index 1fe1c67..2a1e589 100644 --- a/text/shape-build.c +++ b/text/shape-build.c @@ -1,3 +1,4 @@ +#define _POSIX_C_SOURCE 200809L /* clock_gettime */ /* Building shaping sections with HarfBuzz (text/shape-build.h): plans as * HarfBuzz's planner compiles them (hb-ot-shape.cc's features, hb-ot-map.cc's * masks and stages), what the section carries, and a check of it against diff --git a/text/upstream.py b/text/upstream.py new file mode 100644 index 0000000..ec0e18b --- /dev/null +++ b/text/upstream.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Bring in Vega's text upstreams, and hold Vega to them: + + python3 text/upstream.py [--harfbuzz VERSION] [--unicode VERSION] [--corpus] [--offline] + +Vega's shaping is HarfBuzz's, case for case, and its Unicode tables are +generated from the Unicode Character Database. When either moves on, this +is how Vega follows (docs/design/004-text.md): + + 1. fetch HarfBuzz VERSION's release (default: the one the build links, + pkg-config) and the UCD of the Unicode version its tables came from + (its src/hb-ucd-table.hh says; --unicode to choose), once, into + build/upstream; + 2. build that HarfBuzz as one static library, from its amalgamated + harfbuzz.cc: no system package, any version; + 3. regenerate text/unicode-data.c (text/unicode.py) and + text/syllabic-data.c (text/syllabic.py: the syllable machines and the + Indic, USE and Arabic tables), and hold the machines to HarfBuzz's own + compiled ones (syllabic.py --check); + 4. build Vega's text tools against that HarfBuzz (build/upstream/build) + and run make check-text: Unicode's conformance tests on that UCD, + random shaping against that HarfBuzz, the layout's promises, the kern + class tests, and HarfBuzz's own test suite, case by case; + 5. with --corpus, every installed font against it on random runs + (text/fonts.py). + +What a new version breaks shows as failing checks, each with the font and +the input (the suite's case, the random run) to port its fix from. The +generated files are left changed for review (git diff). One JSON line at +the end: the versions, what changed, how each check went; exit 1 when one +failed. --offline uses only what build/upstream already holds. +""" +import argparse +import json +import re +import subprocess +import sys +import tarfile +import urllib.request +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +UPSTREAM = ROOT / 'build' / 'upstream' + + +def fetch(url, into, offline): + if into.exists(): + return + if offline: + sys.exit(f'upstream: {into} is not here, and --offline') + print(f'fetching {url}', file=sys.stderr) + part = into.with_name(into.name + '.part') + with urllib.request.urlopen(url, timeout=300) as response, open(part, 'wb') as out: + while chunk := response.read(1 << 20): + out.write(chunk) + part.rename(into) + + +def harfbuzz(version, offline): + """The release's source, and a static library built from it.""" + source = UPSTREAM / f'harfbuzz-{version}' + if not (source / 'src' / 'harfbuzz.cc').exists(): + archive = UPSTREAM / f'harfbuzz-{version}.tar.xz' + fetch(f'https://github.com/harfbuzz/harfbuzz/releases/download/{version}/harfbuzz-{version}.tar.xz', archive, + offline) + with tarfile.open(archive) as tar: + tar.extractall(UPSTREAM, filter='data') + library = source / 'libharfbuzz.a' + if not library.exists(): + print(f'building HarfBuzz {version}', file=sys.stderr) + subprocess.run(['c++', '-O2', '-fPIC', '-std=c++17', '-DNDEBUG', '-c', str(source / 'src' / 'harfbuzz.cc'), + '-o', str(source / 'harfbuzz.o')], check=True) + subprocess.run(['ar', 'rcs', str(library), str(source / 'harfbuzz.o')], check=True) + return source, library + + +def ucd(version, offline): + directory = UPSTREAM / f'ucd-{version}' + if not (directory / 'ReadMe.txt').exists(): + archive = UPSTREAM / f'UCD-{version}.zip' + fetch(f'https://www.unicode.org/Public/{version}/ucd/UCD.zip', archive, offline) + with zipfile.ZipFile(archive) as z: + z.extractall(directory) + return directory + + +def run(name, command, results, **kwargs): + """A step: its last JSON line (if any) and whether it passed.""" + print(f'== {name}', file=sys.stderr) + done = subprocess.run(command, capture_output=True, text=True, cwd=ROOT, **kwargs) + sys.stderr.write(done.stderr[-4000:]) + reports = [json.loads(line) for line in done.stdout.splitlines() if line.startswith('{')] + results[name] = {'passed': done.returncode == 0, **({'report': reports} if reports else {})} + if done.returncode: + sys.stderr.write(done.stdout[-4000:]) + return done + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--harfbuzz') + parser.add_argument('--unicode') + parser.add_argument('--corpus', action='store_true') + parser.add_argument('--offline', action='store_true') + args = parser.parse_args() + UPSTREAM.mkdir(parents=True, exist_ok=True) + version = args.harfbuzz or subprocess.run(['pkg-config', '--modversion', 'harfbuzz'], capture_output=True, + text=True, check=True).stdout.strip() + source, library = harfbuzz(version, args.offline) + named = re.search(r'Unicode (\d+\.\d+\.\d+)', (source / 'src' / 'hb-ucd-table.hh').read_text()) + unicode = args.unicode or named.group(1) + database = ucd(unicode, args.offline) + + results = {} + generated = ['text/unicode-data.c', 'text/syllabic-data.c'] + with open(ROOT / 'text' / 'unicode-data.c', 'w') as out: + results['unicode.py'] = {'passed': subprocess.run([sys.executable, 'text/unicode.py', str(database)], + stdout=out, cwd=ROOT).returncode == 0} + with open(ROOT / 'text' / 'syllabic-data.c', 'w') as out: + results['syllabic.py'] = {'passed': subprocess.run([sys.executable, 'text/syllabic.py', str(source / 'src')], + stdout=out, cwd=ROOT).returncode == 0} + run('syllabic.py --check', [sys.executable, 'text/syllabic.py', str(source / 'src'), '--check'], results) + + build = UPSTREAM / 'build' + flags = [f'BUILD={build}', f'HARFBUZZ_CFLAGS=-I{source / "src"}', + f'HARFBUZZ_LIBS={library} -lstdc++ -lm', f'UCD={database}', f'HARFBUZZ_SRC={source}'] + run('check-text', ['make', '-s', 'check-text'] + flags, results) + if args.corpus: + run('fonts', [sys.executable, 'text/fonts.py', str(build / 'vega-symbols')], results) + + changed = subprocess.run(['git', 'diff', '--stat', '--'] + generated, capture_output=True, text=True, + cwd=ROOT).stdout.strip().splitlines() + print(json.dumps({'harfbuzz': version, 'unicode': unicode, 'changed': changed[:-1] if changed else [], + 'checks': results})) + return 0 if all(r['passed'] for r in results.values()) else 1 + + +if __name__ == '__main__': + sys.exit(main()) |