1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
#!/usr/bin/env python3
"""The Unicode properties Vega's text needs, as C tables (text/unicode-data.c),
from the Unicode Character Database (15.1 on Debian: /usr/share/unicode).
python3 text/unicode.py [UCD directory] > text/unicode-data.c
A property of every code point is a two-stage table: the code point's upper
bits pick a block of 256 values (blocks that repeat are kept once). Sparse
properties (paired brackets, mirrors) are sorted arrays, searched by code
point. The output is deterministic: the same UCD gives the same file.
"""
import re
import sys
from pathlib import Path
UCD = Path(sys.argv[1] if len(sys.argv) > 1 else '/usr/share/unicode')
BIDI = ['L', 'R', 'AL', 'EN', 'ES', 'ET', 'AN', 'CS', 'NSM', 'BN', 'B', 'S', 'WS', 'ON',
'LRE', 'LRO', 'RLE', 'RLO', 'PDF', 'LRI', 'RLI', 'FSI', 'PDI']
LONG = {'Left_To_Right': 'L', 'Right_To_Left': 'R', 'Arabic_Letter': 'AL', 'European_Number': 'EN',
'European_Separator': 'ES', 'European_Terminator': 'ET', 'Arabic_Number': 'AN', 'Common_Separator': 'CS',
'Nonspacing_Mark': 'NSM', 'Boundary_Neutral': 'BN', 'Paragraph_Separator': 'B', 'Segment_Separator': 'S',
'White_Space': 'WS', 'Other_Neutral': 'ON', 'Left_To_Right_Embedding': 'LRE',
'Left_To_Right_Override': 'LRO', 'Right_To_Left_Embedding': 'RLE', 'Right_To_Left_Override': 'RLO',
'Pop_Directional_Format': 'PDF', 'Left_To_Right_Isolate': 'LRI', 'Right_To_Left_Isolate': 'RLI',
'First_Strong_Isolate': 'FSI', 'Pop_Directional_Isolate': 'PDI'}
LAST = 0x10FFFF
def ranges(path):
"""(first, last, value, missing) of each line of a UCD file, @missing lines included."""
for line in (UCD / path).read_text(encoding='utf-8').splitlines():
missing = line.startswith('# @missing:')
body = line[len('# @missing:'):] if missing else line.split('#', 1)[0]
if not body.strip():
continue
fields = [f.strip() for f in body.split(';')]
span = fields[0].split('..')
yield int(span[0], 16), int(span[-1], 16), fields[1:], missing
def derived(path, names, long_names=None):
"""A value for every code point: the defaults of the @missing lines, then the data."""
values = [0] * (LAST + 1)
lines = list(ranges(path))
for first, last, fields, missing in lines:
if missing:
name = (long_names or {}).get(fields[0], fields[0])
values[first:last + 1] = [names.index(name)] * (last - first + 1)
for first, last, fields, missing in lines:
if not missing:
values[first:last + 1] = [names.index(fields[0])] * (last - first + 1)
return values
def two_stage(name, values):
"""C for a two-stage table of u8 values, and the lookup's name."""
blocks, index, stage1 = [], {}, []
for start in range(0, LAST + 1, 256):
block = tuple(values[start:start + 256])
if block not in index:
index[block] = len(blocks)
blocks.append(block)
stage1.append(index[block])
out = [f'static const uint16_t {name}_blocks[{len(stage1)}] = {{']
for i in range(0, len(stage1), 16):
out.append(' ' + ', '.join(str(v) for v in stage1[i:i + 16]) + ',')
out.append('};')
out.append(f'static const uint8_t {name}_values[{len(blocks) * 256}] = {{')
for block in blocks:
for i in range(0, 256, 32):
out.append(' ' + ','.join(str(v) for v in block[i:i + 32]) + ',')
out.append('};')
return '\n'.join(out), len(stage1) * 2 + len(blocks) * 256
def main():
bidi = derived('extracted/DerivedBidiClass.txt', BIDI, LONG)
table, size = two_stage('bidi', bidi)
brackets = []
for first, _, fields, missing in ranges('BidiBrackets.txt'):
if not missing:
brackets.append((first, int(fields[0], 16), 1 if fields[1] == 'o' else 2))
mirrors = [(first, int(fields[0], 16)) for first, _, fields, missing in ranges('BidiMirroring.txt') if not missing]
version = re.search(r'(\d+\.\d+\.\d+)', (UCD / 'ReadMe.txt').read_text()) if (UCD / 'ReadMe.txt').exists() else None
print('/* Generated by text/unicode.py from the Unicode Character Database'
f'{" " + version.group(1) if version else ""}: do not edit. */')
print('#include "text.h"')
print()
print(f'/* Bidi_Class ({size} bytes). */')
print(table)
print()
print('VegaBidiClass vega_bidi_class(uint32_t code) {')
print(' return code > 0x10FFFF ? VEGA_BIDI_L : (VegaBidiClass)bidi_values[bidi_blocks[code >> 8] * 256u + (code & 255)];')
print('}')
print()
print(f'/* Bidi_Paired_Bracket and Bidi_Paired_Bracket_Type (1 open, 2 close), by code point. */')
print(f'static const uint32_t brackets[{len(brackets)}][3] = {{')
for code, pair, kind in sorted(brackets):
print(f' {{0x{code:04X}, 0x{pair:04X}, {kind}}},')
print('};')
print()
print(f'/* Bidi_Mirroring_Glyph, by code point. */')
print(f'static const uint32_t mirrors[{len(mirrors)}][2] = {{')
for code, mirror in sorted(mirrors):
print(f' {{0x{code:04X}, 0x{mirror:04X}}},')
print('};')
print('''
/* The row of a sorted table of rows whose first value is code, or -1. */
static long find(const uint32_t *rows, size_t count, size_t width, uint32_t code) {
size_t low = 0, high = count;
while (low < high) {
size_t middle = (low + high) / 2;
if (rows[middle * width] < code) low = middle + 1;
else high = middle;
}
return low < count && rows[low * width] == code ? (long)low : -1;
}
int vega_bracket(uint32_t code, uint32_t *pair) {
long row = find(&brackets[0][0], sizeof(brackets) / sizeof(*brackets), 3, code);
if (row < 0) return 0;
*pair = brackets[row][1];
return (int)brackets[row][2];
}
uint32_t vega_mirror(uint32_t code) {
long row = find(&mirrors[0][0], sizeof(mirrors) / sizeof(*mirrors), 2, code);
return row < 0 ? 0 : mirrors[row][1];
}''')
if __name__ == '__main__':
main()
|
