1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
#!/usr/bin/env python3
"""Bring in Vega's text upstreams, and hold Vega to them:
python3 text/upstream.py [--harfbuzz VERSION] [--unicode VERSION] [--corpus] [--offline]
Vega's shaping is HarfBuzz's, case for case, and its Unicode tables are
generated from the Unicode Character Database. When either moves on, this
is how Vega follows (docs/design/004-text.md):
1. fetch HarfBuzz VERSION's release (default: the one the build links,
pkg-config) and the UCD of the Unicode version its tables came from
(its src/hb-ucd-table.hh says; --unicode to choose), once, into
build/upstream;
2. build that HarfBuzz as one static library, from its amalgamated
harfbuzz.cc: no system package, any version;
3. regenerate text/unicode-data.c (text/unicode.py) and
text/syllabic-data.c (text/syllabic.py: the syllable machines and the
Indic, USE and Arabic tables), and hold the machines to HarfBuzz's own
compiled ones (syllabic.py --check);
4. build Vega's text tools against that HarfBuzz (build/upstream/build)
and run make check-text: Unicode's conformance tests on that UCD,
random shaping against that HarfBuzz, the layout's promises, the kern
class tests, and HarfBuzz's own test suite, case by case;
5. with --corpus, every installed font against it on random runs
(text/fonts.py).
What a new version breaks shows as failing checks, each with the font and
the input (the suite's case, the random run) to port its fix from. The
generated files are left changed for review (git diff). One JSON line at
the end: the versions, what changed, how each check went; exit 1 when one
failed. --offline uses only what build/upstream already holds.
"""
import argparse
import json
import re
import subprocess
import sys
import tarfile
import urllib.request
import zipfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
UPSTREAM = ROOT / 'build' / 'upstream'
def fetch(url, into, offline):
if into.exists():
return
if offline:
sys.exit(f'upstream: {into} is not here, and --offline')
print(f'fetching {url}', file=sys.stderr)
part = into.with_name(into.name + '.part')
with urllib.request.urlopen(url, timeout=300) as response, open(part, 'wb') as out:
while chunk := response.read(1 << 20):
out.write(chunk)
part.rename(into)
def harfbuzz(version, offline):
"""The release's source, and a static library built from it."""
source = UPSTREAM / f'harfbuzz-{version}'
if not (source / 'src' / 'harfbuzz.cc').exists():
archive = UPSTREAM / f'harfbuzz-{version}.tar.xz'
fetch(f'https://github.com/harfbuzz/harfbuzz/releases/download/{version}/harfbuzz-{version}.tar.xz', archive,
offline)
with tarfile.open(archive) as tar:
tar.extractall(UPSTREAM, filter='data')
library = source / 'libharfbuzz.a'
if not library.exists():
print(f'building HarfBuzz {version}', file=sys.stderr)
subprocess.run(['c++', '-O2', '-fPIC', '-std=c++17', '-DNDEBUG', '-c', str(source / 'src' / 'harfbuzz.cc'),
'-o', str(source / 'harfbuzz.o')], check=True)
subprocess.run(['ar', 'rcs', str(library), str(source / 'harfbuzz.o')], check=True)
return source, library
def ucd(version, offline):
directory = UPSTREAM / f'ucd-{version}'
if not (directory / 'ReadMe.txt').exists():
archive = UPSTREAM / f'UCD-{version}.zip'
fetch(f'https://www.unicode.org/Public/{version}/ucd/UCD.zip', archive, offline)
with zipfile.ZipFile(archive) as z:
z.extractall(directory)
return directory
def run(name, command, results, **kwargs):
"""A step: its last JSON line (if any) and whether it passed."""
print(f'== {name}', file=sys.stderr)
done = subprocess.run(command, capture_output=True, text=True, cwd=ROOT, **kwargs)
sys.stderr.write(done.stderr[-4000:])
reports = [json.loads(line) for line in done.stdout.splitlines() if line.startswith('{"')]
results[name] = {'passed': done.returncode == 0, **({'report': reports} if reports else {})}
if done.returncode and not name.endswith('.py'):
sys.stderr.write(done.stdout[-4000:])
return done
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--harfbuzz')
parser.add_argument('--unicode')
parser.add_argument('--corpus', action='store_true')
parser.add_argument('--offline', action='store_true')
args = parser.parse_args()
UPSTREAM.mkdir(parents=True, exist_ok=True)
version = args.harfbuzz or subprocess.run(['pkg-config', '--modversion', 'harfbuzz'], capture_output=True,
text=True, check=True).stdout.strip()
source, library = harfbuzz(version, args.offline)
named = re.search(r'Unicode (\d+\.\d+\.\d+)', (source / 'src' / 'hb-ucd-table.hh').read_text())
unicode = args.unicode or named.group(1)
database = ucd(unicode, args.offline)
results = {}
generated = ['text/unicode-data.c', 'text/syllabic-data.c']
for name, target, given in (('unicode.py', 'unicode-data.c', database), ('syllabic.py', 'syllabic-data.c',
source / 'src')):
# a generator that fails leaves its file as it was
made = run(name, [sys.executable, f'text/{name}', str(given)], results)
if not made.returncode:
(ROOT / 'text' / target).write_text(made.stdout)
run('syllabic.py --check', [sys.executable, 'text/syllabic.py', str(source / 'src'), '--check'], results)
if not all(r['passed'] for r in results.values()):
# the generators do not follow this version yet: what to port first
print(json.dumps({'harfbuzz': version, 'unicode': unicode, 'checks': results}))
return 1
build = UPSTREAM / 'build'
flags = [f'BUILD={build}', f'HARFBUZZ_CFLAGS=-I{source / "src"}',
f'HARFBUZZ_LIBS={library} -lstdc++ -lm', f'UCD={database}', f'HARFBUZZ_SRC={source}']
run('check-text', ['make', '-s', 'check-text'] + flags, results)
if args.corpus:
run('fonts', [sys.executable, 'text/fonts.py', str(build / 'vega-symbols')], results)
changed = subprocess.run(['git', 'diff', '--stat', '--'] + generated, capture_output=True, text=True,
cwd=ROOT).stdout.strip().splitlines()
print(json.dumps({'harfbuzz': version, 'unicode': unicode, 'changed': changed[:-1] if changed else [],
'checks': results}))
return 0 if all(r['passed'] for r in results.values()) else 1
if __name__ == '__main__':
sys.exit(main())
|
