Shape Syriac, stretch with stch, and fall back for fonts without Arabic forms

The rest of HarfBuzz 10.2's Arabic shaper. Joining types now come from
HarfBuzz's own table (Unicode 16, with Syriac's Alaph and Dalath-Rish
groups the state machine's last two columns use), the general category
deciding where it has none, so Syriac shapes with fin2, fin3 and med2.
What stch multiplies is recorded and, after positioning, stretched under
the word before it, repeating pieces copied as often as fit. A font with
presentation forms but no Arabic features gets HarfBuzz's fallback
lookups, which the build tool writes as OpenType and the runtime applies
at their pause like any GSUB lookup.

The generator now reads HarfBuzz's generated tables through their lookup
functions' ranges and offsets: rows whose range starts mid-row have
blank cells in front, which the row comments alone miscounted (Arabic
Supplement); the Indic table, read again this way, did not change.

Noto Sans Syriac, FreeSans's Syriac, Unifont's Arabic and Noto Sans
Arabic shape as HarfBuzz does at 3,000 runs a plan; no installed font
is refused any more (213 fonts, 1,822 plans).

1 files changed, 98 insertions, 7 deletions

diff --git a/text/syllabic.py b/text/syllabic.py
index 4a79869..b4de81a 100644
--- a/text/syllabic.py
+++ b/text/syllabic.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""The tables HarfBuzz's syllabic shapers read, as C (text/syllabic-data.c).
+"""The tables HarfBuzz's complex shapers read, as C (text/syllabic-data.c).
 
     python3 text/syllabic.py HARFBUZZ_SRC > text/syllabic-data.c
     python3 text/syllabic.py HARFBUZZ_SRC --check
@@ -16,6 +16,10 @@ HARFBUZZ_SRC is HarfBuzz 10.2.0's src/ directory; what is read from it:
   shaper-use-table.hh, whose arrays and lookup (gen-use-table.py's) are
   carried over as they are, the categories named in them resolved.
 
+- Arabic's: each character's joining type (with Syriac's Alaph and
+  Dalath-Rish groups), and the presentation forms and ligatures fallback
+  shaping builds lookups from (hb-ot-shaper-arabic-table.hh).
+
 - The syllable machines: hb-ot-shaper-*-machine.rl, the Ragel grammars.
   This compiles each to a DFA with Ragel's scanner semantics (the longest
   match from each position, ties to the pattern listed first), its symbols
@@ -242,6 +246,29 @@ def scan(table, accept, symbols):
     return found
 
 
+# ---- HarfBuzz's generated tables: a flat array (rows commented with the
+# code point they start at, a row's first cells left blank where a range
+# starts inside it) and a lookup of ranges into it
+
+def ranged_table(text, array, function, cell):
+    """Code point to cell for a table whose lookup function returns
+    array[u - first + offset] over ranges; cell() reads one cell's text."""
+    body = text[text.index(f'{array}[] =') :]
+    body = body[body.index('{') + 1:body.index('\n};')]
+    body = re.sub(r'/\*.*?\*/', ' ', body, flags=re.S)
+    body = re.sub(r'#define[^\n]*', ' ', body)
+    cells = cell(body)
+    offsets = {int(k, 16): int(v) for k, v in re.findall(rf'#define {array.split("_")[0]}_offset_0x([0-9a-f]+)u (\d+)', text)}
+    lookup = text[text.index(function):]
+    table = {}
+    for first, last, key in re.findall(r'hb_in_range<hb_codepoint_t> \(u, 0x([0-9A-F]+)u, 0x([0-9A-F]+)u\)\) return '
+                                       rf'{array}\[u - 0x[0-9A-F]+u \+ \w+_offset_0x([0-9a-f]+)u\]', lookup):
+        first, last, at = int(first, 16), int(last, 16), offsets[int(key, 16)]
+        for u in range(first, last + 1):
+            table[u] = cells[at + u - first]
+    return table, lookup
+
+
 # ---- The Indic table
 
 def indic_table(grammars):
@@ -258,12 +285,8 @@ def indic_table(grammars):
     def cell(s, m):
         return short['_OT_' + s], short['_POS_' + m]
 
-    cells = {}
-    table = text[text.index('static const uint16_t indic_table[]'):text.index('}; /* Table items:')]
-    for start, row in re.findall(r'/\* ([0-9A-F]{4,5}) \*/(.*)', table):
-        for k, (s, m) in enumerate(re.findall(r'_\((\w+),(\w+)\)', row)):
-            cells[int(start, 16) + k] = cell(s, m)
-    lookup = text[text.index('hb_indic_get_categories (hb_codepoint_t u)'):]
+    cells, lookup = ranged_table(text, 'indic_table', 'hb_indic_get_categories (hb_codepoint_t u)',
+                                 lambda body: [cell(s, m) for s, m in re.findall(r'_\((\w+),(\w+)\)', body)])
     for code, s, m in re.findall(r'u == 0x([0-9A-F]+)u\)\) return _\((\w+),(\w+)\);', lookup):
         cells[int(code, 16)] = cell(s, m)
     default = cell(*re.search(r'return _\((\w+),(\w+)\);\s*\}', lookup).groups())
@@ -297,6 +320,34 @@ def use_table(grammar):
     return arrays, expression[:-1] + str(grammar.symbols['O'])
 
 
+def arabic_tables():
+    """hb-ot-shaper-arabic-table.hh: joining type by code point (U 0, L 1,
+    R 2, D and C 3, Alaph 4, Dalath-Rish 5, T 7, X 8 elsewhere); the
+    presentation forms of U+0621..06D3 (init, medi, fina, isol); the
+    ligature sets, each as (first, ligature, components)."""
+    text = (HB / 'hb-ot-shaper-arabic-table.hh').read_text()
+    values = {'U': 0, 'L': 1, 'R': 2, 'D': 3, 'C': 3, 'A': 4, 'DR': 5, 'T': 7, 'X': 8}
+    joining, _ = ranged_table(text, 'joining_table', 'joining_type (hb_codepoint_t u)',
+                              lambda body: [values[n] for n in re.findall(r'[A-Z]+', body)])
+    shaping = [tuple(int(v, 16) for v in re.findall(r'0x([0-9A-F]{4})u', row))
+               for row in re.findall(r'\{(0x[^}]*)\}, /\* U\+', text[text.index('shaping_table[][4]'):])]
+    first = int(re.search(r'SHAPING_TABLE_FIRST\s+0x([0-9A-F]+)u', text).group(1), 16)
+    last = int(re.search(r'SHAPING_TABLE_LAST\s+0x([0-9A-F]+)u', text).group(1), 16)
+    assert len(shaping) == last - first + 1 and all(len(r) == 4 for r in shaping)
+    ligatures = {}
+    for name in ('ligature_table', 'ligature_mark_table', 'ligature_3_table'):
+        body = text[text.index(f'}} {name}[] =') :]
+        body = body[:body.index('\n};')]
+        sets = []
+        for first_u, entries in re.findall(r'\{ 0x([0-9A-F]+)u, \{(.*?)\}\},', body, re.S):
+            for components, ligature in re.findall(r'\{ \{([^}]*)\}, 0x([0-9A-F]+)u ?\}', entries):
+                sets.append((int(first_u, 16), int(ligature, 16),
+                             [int(c, 16) for c in re.findall(r'0x([0-9A-F]+)u', components)]))
+        assert sets, name
+        ligatures[name] = sets
+    return joining, (first, shaping), ligatures
+
+
 # ---- Checking against Ragel's machines (hb-ot-shaper-*-machine.hh)
 
 def ragel(name):
@@ -476,6 +527,46 @@ def main():
     out.append('static unsigned use_b4(const uint8_t *a, unsigned i) { return (a[i >> 1] >> ((i & 1u) << 2)) & 15u; }')
     out.append(f'uint8_t vega_use_category(uint32_t u) {{ return (uint8_t)({expression}); }}')
     out.append('')
+    joining, (first, shaping), ligatures = arabic_tables()
+    runs, flat = [], []
+    for a, b in runs_of(joining):
+        runs.append((a, b, len(flat)))
+        flat.extend(joining.get(code, 8) for code in range(a, b + 1))
+    out.append('/* Arabic joining types (hb-ot-shaper-arabic-table.hh, Unicode 16): U 0, L 1, R 2, D and C 3,')
+    out.append(' * Alaph 4, Dalath-Rish 5, T 7; 8 where the general category decides. */')
+    out.append(f'static const uint8_t joining_cells[{len(flat)}] = {{')
+    for k in range(0, len(flat), 32):
+        out.append('    ' + ', '.join(str(v) for v in flat[k:k + 32]) + ',')
+    out.append('};')
+    out.append(f'static const struct {{ uint32_t first, last, at; }} joining_runs[{len(runs)}] = {{')
+    for a, b, at in runs:
+        out.append(f'    {{0x{a:04X}, 0x{b:04X}, {at}}},')
+    out.append('};')
+    out.append('uint8_t vega_arabic_joining(uint32_t u) {')
+    out.append(f'    for (unsigned i = 0; i < {len(runs)} && u >= joining_runs[i].first; ++i)')
+    out.append('        if (u <= joining_runs[i].last) return joining_cells[joining_runs[i].at + u - joining_runs[i].first];')
+    out.append('    return 8;')
+    out.append('}')
+    out.append('')
+    out.append(f'/* Presentation forms of U+{first:04X}..U+{first + len(shaping) - 1:04X}: init, medi, fina, isol. */')
+    out.append(f'const uint16_t vega_arabic_forms[{len(shaping)}][4] = {{')
+    for row in shaping:
+        out.append('    {' + ', '.join(f'0x{v:04X}' for v in row) + '},')
+    out.append('};')
+    out.append('/* Ligatures fallback shaping forms: three components, two, and marks. */')
+    for name, cname in (('ligature_3_table', 'three'), ('ligature_table', 'two'), ('ligature_mark_table', 'marks')):
+        sets = ligatures[name]
+        out.append(f'static const VegaArabicLigature ligatures_{cname}[{len(sets)}] = {{')
+        for first_u, ligature, components in sets:
+            cs = components + [0] * (2 - len(components))
+            out.append(f'    {{0x{first_u:04X}, 0x{ligature:04X}, {{0x{cs[0]:04X}, 0x{cs[1]:04X}}}}},')
+        out.append('};')
+    out.append('const VegaArabicLigatures vega_arabic_ligatures[3] = {')
+    for name, cname, n in (('ligature_3_table', 'three', 3), ('ligature_table', 'two', 2),
+                           ('ligature_mark_table', 'marks', 2)):
+        out.append(f'    {{ligatures_{cname}, {len(ligatures[name])}, {n}}},')
+    out.append('};')
+    out.append('')
     out.append('/* The categories syllabic.h names are the machines\' own. */')
     for name, prefix in (('indic', 'VEGA_OT_'), ('khmer', 'VEGA_OT_'), ('myanmar', 'VEGA_OT_'), ('use', 'VEGA_USE_')):
         for symbol, value in sorted(grammars[name].symbols.items()):