diff options
| -rw-r--r-- | Makefile | 13 | ||||
| -rw-r--r-- | symbols/symbols-tool.c | 126 | ||||
| -rw-r--r-- | text/harfbuzz-suite.known | 40 | ||||
| -rw-r--r-- | text/harfbuzz-suite.py | 152 |
4 files changed, 328 insertions, 3 deletions
@@ -15,7 +15,7 @@ LIBPURE = $(PURE)/build/libpure.a SURFACES = surfaces/screen.c surfaces/draw.c surfaces/width.c surfaces/web.c surfaces/pixels.c surfaces/display.c \ surfaces/input.c surfaces/hangul.c surfaces/dbus.c surfaces/access.c surfaces/main.c -.PHONY: all check FORCE +.PHONY: all check check-harfbuzz FORCE all: $(BUILD)/vega $(BUILD)/vega-symbols $(BUILD)/vega-surface.so $(BUILD): @@ -76,7 +76,16 @@ $(BUILD)/test-layout: text/test-layout.c $(LAYOUT) text/layout.h text/shape.h te UCD ?= /usr/share/unicode SHAPED_FONT ?= /usr/share/fonts/truetype/dejavu/DejaVuSans.ttf SHAPED_CODES = 0x20-0x7e,0xa0-0x24f,0x300-0x36f,0x370-0x3ff,0x400-0x4ff,0x591-0x5f4,0x600-0x6ff,0x2000-0x206f -check: $(BUILD)/vega $(BUILD)/vega-symbols $(BUILD)/vega-surface.so $(BUILD)/test-text $(BUILD)/test-layout +# HarfBuzz's own shaping tests, where a HarfBuzz release's source is +# (HARFBUZZ_SRC: the same version the build links): each case shaped by +# Vega and by HarfBuzz must agree, but for those text/harfbuzz-suite.known +# lists. +HARFBUZZ_SRC ?= +check-harfbuzz: $(BUILD)/vega-symbols + if [ -d "$(HARFBUZZ_SRC)/test/shape/data" ]; then python3 text/harfbuzz-suite.py $(HARFBUZZ_SRC) $(BUILD)/vega-symbols; \ + else echo "no HarfBuzz source at HARFBUZZ_SRC: its test suite skipped"; fi + +check: $(BUILD)/vega $(BUILD)/vega-symbols $(BUILD)/vega-surface.so $(BUILD)/test-text $(BUILD)/test-layout check-harfbuzz if [ -f $(UCD)/BidiTest.txt ]; then $(BUILD)/test-text $(UCD); else echo "no UCD at $(UCD): text tests skipped"; fi if [ -f $(SHAPED_FONT) ]; then $(BUILD)/vega-symbols build --font $(SHAPED_FONT) --codes $(SHAPED_CODES) \ --shape Latn,Grek,Cyrl,Hebr,Arab,Zyyy --runs 5000 -o $(BUILD)/shaped.vsym && \ diff --git a/symbols/symbols-tool.c b/symbols/symbols-tool.c index 1c0f198..ac1ad99 100644 --- a/symbols/symbols-tool.c +++ b/symbols/symbols-tool.c @@ -4,6 +4,7 @@ * vega-symbols build [--font FONT.ttf [--face N] --codes 0x20-0x7e,... [--shape Latn,Arab,...]]... [--svg CODE=ICON.svg ...] * [--units N] [--runs N] -o OUT.vsym * vega-symbols shape-check FONT.ttf OUT.vsym [--face N] [--runs N] [--seed N] + * vega-symbols shape-cases FONT.ttf [--face N] < CASES * vega-symbols compare FONT.ttf OUT.vsym --sizes 12,16,32,64 * vega-symbols info OUT.vsym * vega-symbols render OUT.vsym CODE PIXELS OUT.pgm @@ -1150,6 +1151,128 @@ static int shape_one(int argc, char **argv) { return 0; } +/* Shape the cases given with a section compiled for them and with + * HarfBuzz: vega-symbols shape-cases FONT.ttf [--face N] < CASES, a case a + * line: ID SCRIPT DIRECTION CODES (an ISO 15924 code or -, ltr, rtl or -, + * hex code points joined by commas). What is - HarfBuzz guesses from the + * codes (hb_buffer_guess_segment_properties; Common when it finds no + * script). A line out per case: ID same; ID differs, then both glyph + * lists; ID failed (no plan); or, when the font's shaping cannot be + * compiled, ID refused and why. For HarfBuzz's own test suite + * (text/harfbuzz-suite.py). */ +typedef struct { char id[160]; uint32_t script, *codes; int direction; size_t count; } Case; + +static void print_glyphs(const char *who, const VegaShaped *g, size_t count) { + printf(" %-8s", who); + for (size_t i = 0; i < count; ++i) + printf(" %u@%u(%d,%d+%d,%d)", g[i].glyph, g[i].cluster, g[i].x_advance, g[i].y_advance, g[i].x_offset, + g[i].y_offset); + printf("\n"); +} + +static int shape_cases(int argc, char **argv) { + if (argc != 3 && !(argc == 5 && !strcmp(argv[3], "--face"))) + fail("usage: vega-symbols shape-cases FONT.ttf [--face N] < CASES"); + hb_blob_t *blob = hb_blob_create_from_file_or_fail(argv[2]); + if (!blob) fail("cannot read the font"); + hb_face_t *face = hb_face_create(blob, argc == 5 ? (unsigned)strtoul(argv[4], NULL, 10) : 0); + hb_font_t *font = hb_font_create(face); + hb_buffer_t *buffer = hb_buffer_create(); + Case *cases = NULL; + size_t count = 0, allocated = 0, script_count = 0; + uint32_t scripts[256]; + char line[65536]; + while (fgets(line, sizeof(line), stdin)) { + char id[160], script[16], direction[16], *codes = malloc(sizeof(line)); + if (!codes || sscanf(line, "%159s %15s %15s %65535s", id, script, direction, codes) != 4) { + free(codes); + continue; + } + if (count == allocated) { + allocated = allocated ? 2 * allocated : 64; + cases = realloc(cases, allocated * sizeof(*cases)); + if (!cases) fail("out of memory"); + } + Case *c = &cases[count++]; + *c = (Case){.count = 0}; + snprintf(c->id, sizeof(c->id), "%s", id); + c->codes = malloc((strlen(codes) / 2 + 2) * sizeof(*c->codes)); + if (!c->codes) fail("out of memory"); + for (char *p = codes; *p;) { + c->codes[c->count++] = (uint32_t)strtoul(p, &p, 16); + if (*p == ',') ++p; + else if (*p) break; + } + free(codes); + /* What is not given, as HarfBuzz guesses it. */ + hb_buffer_clear_contents(buffer); + hb_buffer_add_codepoints(buffer, c->codes, (int)c->count, 0, (int)c->count); + if (strcmp(script, "-")) hb_buffer_set_script(buffer, hb_script_from_string(script, -1)); + if (!strcmp(direction, "ltr")) hb_buffer_set_direction(buffer, HB_DIRECTION_LTR); + if (!strcmp(direction, "rtl")) hb_buffer_set_direction(buffer, HB_DIRECTION_RTL); + hb_buffer_guess_segment_properties(buffer); + hb_script_t guessed = hb_buffer_get_script(buffer); + if (guessed == HB_SCRIPT_INVALID || guessed == HB_SCRIPT_UNKNOWN) guessed = HB_SCRIPT_COMMON; + c->script = hb_script_to_iso15924_tag(guessed); + c->direction = hb_buffer_get_direction(buffer) == HB_DIRECTION_RTL ? VEGA_SHAPE_RTL : VEGA_SHAPE_LTR; + bool listed = false; + for (size_t k = 0; k < script_count && !listed; ++k) listed = scripts[k] == c->script; + if (!listed && script_count < 256) scripts[script_count++] = c->script; + } + VegaSection section = {0}; + VegaShaping shaping; + char error[256]; + bool compiled = vega_shaping_compile(face, scripts, script_count, NULL, 0, §ion, NULL, error, sizeof(error)) && + vega_shaping_open(&shaping, section.bytes, section.size, error, sizeof(error)); + VegaShaper *shaper = vega_shaper_new(); + for (size_t i = 0; i < count; ++i) { + Case *c = &cases[i]; + if (!compiled) { + printf("%s refused %s\n", c->id, error); + continue; + } + const VegaShaped *ours; + long n = vega_shape(shaper, &shaping, c->codes, c->count, c->script, c->direction, &ours); + hb_buffer_clear_contents(buffer); + hb_buffer_add_codepoints(buffer, c->codes, (int)c->count, 0, (int)c->count); + hb_buffer_set_direction(buffer, c->direction == VEGA_SHAPE_RTL ? HB_DIRECTION_RTL : HB_DIRECTION_LTR); + hb_buffer_set_script(buffer, hb_script_from_iso15924_tag(c->script)); + hb_shape(font, buffer, NULL, 0); + unsigned m; + hb_glyph_info_t *info = hb_buffer_get_glyph_infos(buffer, &m); + hb_glyph_position_t *pos = hb_buffer_get_glyph_positions(buffer, &m); + bool same = n == (long)m; + for (unsigned k = 0; same && k < m; ++k) + same = ours[k].glyph == info[k].codepoint && ours[k].cluster == info[k].cluster && + ours[k].x_advance == pos[k].x_advance && ours[k].y_advance == pos[k].y_advance && + ours[k].x_offset == pos[k].x_offset && ours[k].y_offset == pos[k].y_offset; + if (n < 0) printf("%s failed %c%c%c%c\n", c->id, (char)(c->script >> 24), (char)(c->script >> 16), + (char)(c->script >> 8), (char)c->script); + else if (same) printf("%s same\n", c->id); + else { + printf("%s differs %c%c%c%c %s\n", c->id, (char)(c->script >> 24), (char)(c->script >> 16), + (char)(c->script >> 8), (char)c->script, c->direction == VEGA_SHAPE_RTL ? "rtl" : "ltr"); + print_glyphs("vega", ours, (size_t)n); + VegaShaped *theirs = malloc((m + 1) * sizeof(*theirs)); + for (unsigned k = 0; theirs && k < m; ++k) + theirs[k] = (VegaShaped){info[k].codepoint, info[k].cluster, pos[k].x_advance, pos[k].y_advance, + pos[k].x_offset, pos[k].y_offset}; + if (theirs) print_glyphs("harfbuzz", theirs, m); + free(theirs); + } + } + if (compiled) vega_shaping_close(&shaping); + vega_shaper_free(shaper); + free(section.bytes); + for (size_t i = 0; i < count; ++i) free(cases[i].codes); + free(cases); + hb_buffer_destroy(buffer); + hb_font_destroy(font); + hb_face_destroy(face); + hb_blob_destroy(blob); + return 0; +} + /* UTF-8 (valid) to code points; how many. */ static size_t decode_utf8(const char *text, uint32_t *out, size_t room) { size_t n = 0; @@ -1242,10 +1365,11 @@ int main(int argc, char **argv) { if (argc > 1 && !strcmp(argv[1], "shape")) return shape_one(argc, argv); if (argc > 1 && !strcmp(argv[1], "build")) return build(argc, argv); if (argc > 1 && !strcmp(argv[1], "shape-check")) return shape_check(argc, argv); + if (argc > 1 && !strcmp(argv[1], "shape-cases")) return shape_cases(argc, argv); if (argc > 1 && !strcmp(argv[1], "compare")) return compare(argc, argv); if (argc > 1 && !strcmp(argv[1], "info")) return info(argc, argv); if (argc > 1 && !strcmp(argv[1], "render")) return render(argc, argv); if (argc > 1 && !strcmp(argv[1], "gpu")) return gpu(argc, argv); - fail("vega-symbols build|shape-check|compare|info|render|gpu ..."); + fail("vega-symbols build|shape-check|shape-cases|shape|paragraph|compare|info|render|gpu ..."); return 2; } diff --git a/text/harfbuzz-suite.known b/text/harfbuzz-suite.known new file mode 100644 index 0000000..a0436bb --- /dev/null +++ b/text/harfbuzz-suite.known @@ -0,0 +1,40 @@ +# Cases of HarfBuzz's test suite (text/harfbuzz-suite.py) known to shape +# otherwise in Vega: a case ID or a pattern of them, then why. + +# AAT (morx, kerx, trak): Apple's shaping tables; Vega shapes with +# OpenType's alone and refuses a font that has them. +text-rendering-tests/MORX-* # AAT +in-house/aat-* # AAT + +# The character map made from what HarfBuzz lists (hb_face_collect_unicodes), +# not from how it maps: a format 13 map it lists cut short, a symbol map +# whose Latin and Arabic it maps to the private use area. +in-house/emoji-clusters.tests:* # cmap format 13 +text-rendering-tests/CMAP-3.tests:1 # cmap +text-rendering-tests/CMAP-3.tests:11 # cmap +in-house/arabic-fallback-shaping.tests:7 # symbol cmap, Arabic PUA +in-house/arabic-fallback-shaping.tests:8 # symbol cmap, Arabic PUA +in-house/arabic-fallback-shaping.tests:9 # symbol cmap, Arabic PUA +in-house/arabic-fallback-shaping.tests:10 # symbol cmap, Arabic PUA +in-house/arabic-fallback-shaping.tests:11 # symbol cmap, Arabic PUA + +# Variation selectors (cmap format 14). +text-rendering-tests/CMAP-1.tests:2 # cmap format 14 +text-rendering-tests/CMAP-1.tests:3 # cmap format 14 +text-rendering-tests/CMAP-2.tests:2 # cmap format 14 + +# Arabic fallback shaping where two characters share a glyph (refused). +in-house/arabic-fallback-shaping.tests:2 +in-house/arabic-fallback-shaping.tests:3 +in-house/arabic-fallback-shaping.tests:4 +in-house/arabic-fallback-shaping.tests:5 +in-house/arabic-fallback-shaping.tests:6 + +# A kern table of class pairs (format 2; refused). +in-house/kern-format2.tests:* + +# Myanmar as Zawgyi encodes it (Qaag; refused). +in-house/myanmar-zawgyi.tests:* + +# No plan for the run. +text-rendering-tests/GSUB-3.tests:1 diff --git a/text/harfbuzz-suite.py b/text/harfbuzz-suite.py new file mode 100644 index 0000000..aa25195 --- /dev/null +++ b/text/harfbuzz-suite.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""HarfBuzz's own shaping tests, as cases for Vega's shaping: + + python3 text/harfbuzz-suite.py HARFBUZZ_SRC VEGA_SYMBOLS [--known FILE] [--show N] + +HARFBUZZ_SRC is a HarfBuzz release's source; its test/shape/data holds +fonts and inputs (in-house, text-rendering-tests, aots), many of them added +when a font or a shaper once went wrong. Their expected outputs are +HarfBuzz's under options Vega has no counterpart for (font functions, +sizes, output formats), so the oracle is HarfBuzz itself run on the same +font and input (vega-symbols shape-cases, linked with the HarfBuzz the +build uses): each case must shape the same, glyph for glyph and unit for +unit. A case asking for what Vega does not do (user features, variations, +a language, context, cluster levels, buffer flags, vertical text) is left +out and counted by why. + +Cases known to differ are listed in the known file (text/harfbuzz-suite.known: +an ID or a pattern of IDs a line, then # and why). The run fails on any +other difference, and names known ones that now agree. It prints one JSON +line: what was compared, what was left out and why. +""" +import argparse +import fnmatch +import json +import shlex +import subprocess +import sys +from collections import Counter, defaultdict +from pathlib import Path + +# Options that change only what hb-shape prints, or how it gets a font's +# metrics (the oracle uses HarfBuzz's own, as Vega's build does), or buffer +# flags about default ignorables: the input still tells. +IGNORED = {'--font-funcs', '--font-size', '--ned', '--no-glyph-names', '--no-positions', '--no-clusters', + '--no-advances', '--single-par', '--show-flags', '--show-extents', '--show-text', '--show-unicode', + '--show-line-num', '--unsafe-to-concat', '--safe-to-insert-tatweel', '--verify', '--utf8-clusters', + '--normalize-glyphs', '--remove-default-ignorables', '--preserve-default-ignorables', '--glyphs', + '--no-glyph-extents'} +VALUED = {'--font-funcs', '--font-size', '--features', '--variations', '--direction', '--language', '--script', + '--unicodes-before', '--unicodes-after', '--cluster-level', '--shaper', '--shapers', '--font-ptem', + '--face-index', '--not-found-variation-selector-glyph', '--invisible-glyph', '--named-instance', + '--font-slant', '--font-bold', '--font-grade'} + + +def cases(root): + """(id, font, face, script, direction, codes) of each case, or (id, None, why) for one left out.""" + data = root / 'test' / 'shape' / 'data' + for tests in sorted(data.glob('*/tests/*.tests')): + for number, line in enumerate(tests.read_text(encoding='utf-8').splitlines(), 1): + if not line.strip() or line.startswith('#'): + continue + case = f'{tests.parent.parent.name}/{tests.name}:{number}' + fields = line.split(';') + if len(fields) < 4: + yield case, None, 'malformed' + continue + font, options, text = fields[0], fields[1], fields[2] + tokens, why = shlex.split(options), None + face, script, direction = 0, '-', '-' + i = 0 + while i < len(tokens) and not why: + name, _, value = tokens[i].partition('=') + if name in VALUED and not _: + i += 1 + value = tokens[i] if i < len(tokens) else '' + i += 1 + if name in IGNORED: + continue + if name == '--direction': + direction = {'l': 'ltr', 'ltr': 'ltr', 'r': 'rtl', 'rtl': 'rtl'}.get(value.lower()) + why = None if direction else 'vertical text' + elif name == '--script': + script = value + elif name == '--face-index': + face = int(value) + else: + why = {'--features': 'user features', '--variations': 'variations', '--language': 'a language', + '--unicodes-before': 'context', '--unicodes-after': 'context', + '--cluster-level': 'cluster levels', '--bot': 'buffer flags', '--eot': 'buffer flags', + '--font-ptem': 'point size (trak)'}.get(name, f'option {name}') + if why: + yield case, None, why + continue + # as hb-shape's --unicodes reads them: hex, U+ or 0x before or not + codes = [int(part[2:] if part[:2].lower() in ('u+', '0x') else part, 16) + for part in text.replace(',', ' ').split()] + if not codes: + yield case, None, 'no text' + continue + yield case, (tests.parent / font).resolve(), face, script, direction, codes + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('harfbuzz', type=Path) + parser.add_argument('tool', type=Path) + parser.add_argument('--known', type=Path, default=Path(__file__).with_name('harfbuzz-suite.known')) + parser.add_argument('--show', type=int, default=10) + args = parser.parse_args() + known = [] + if args.known.exists(): + for line in args.known.read_text(encoding='utf-8').splitlines(): + pattern = line.split('#', 1)[0].strip() + if pattern: + known.append(pattern) + left_out, by_font = Counter(), defaultdict(list) + for case in cases(args.harfbuzz): + if case[1] is None: + left_out[case[2]] += 1 + else: + by_font[(case[1], case[2])].append(case) + outcome, details = {}, {} + for (font, face), group in sorted(by_font.items()): + if not font.exists(): # a system font of another system (macOS) + left_out['font not here'] += len(group) + continue + lines = ''.join(f'{case[0]} {case[3]} {case[4]} {",".join(f"{c:04X}" for c in case[5])}\n' for case in group) + command = [str(args.tool), 'shape-cases', str(font)] + (['--face', str(face)] if face else []) + run = subprocess.run(command, input=lines, capture_output=True, text=True, timeout=600) + if run.returncode: + for case in group: + outcome[case[0]] = f'tool failed: {run.stderr.strip()[:80]}' + continue + last = None + for line in run.stdout.splitlines(): + if line.startswith(' '): + details[last] = details.get(last, '') + line + '\n' + continue + case, verdict = line.split(' ', 1) + outcome[case], last = verdict, case + for case in group: + outcome.setdefault(case[0], 'no answer') + counts = Counter(verdict.split(' ', 1)[0] for verdict in outcome.values()) + is_known = lambda case: any(fnmatch.fnmatchcase(case, pattern) for pattern in known) + unexpected = sorted(case for case, verdict in outcome.items() if verdict != 'same' and not is_known(case)) + now_same = sorted(case for case, verdict in outcome.items() if verdict == 'same' and is_known(case) and + case in known) + for case in unexpected[:args.show]: + print(f'{case}: {outcome[case]}', file=sys.stderr) + print(details.get(case, ''), end='', file=sys.stderr) + for case in now_same: + print(f'{case}: known to differ, now the same', file=sys.stderr) + print(json.dumps({'cases': len(outcome) + sum(left_out.values()), 'compared': len(outcome), 'same': counts['same'], + 'differ': counts['differs'], 'failed': counts['failed'], 'refused': counts['refused'], + 'other': len(outcome) - counts['same'] - counts['differs'] - counts['failed'] - counts['refused'], + 'unexpected': len(unexpected), 'known_now_same': len(now_same), + 'left_out': dict(left_out.most_common())})) + return 1 if unexpected else 0 + + +if __name__ == '__main__': + sys.exit(main()) |