From 0d2e04f1c9d05a598deb443ba067ef61e0b9cab3 Mon Sep 17 00:00:00 2001 From: eunhhu Date: Mon, 28 Sep 2026 12:57:44 +0100 Subject: Break text into lines, graphemes, words and sentences as Unicode's tests require UAX #14 line breaking (LB1-LB31, numbers tailored as Unicode's tests are) and UAX #29's grapheme cluster, word and sentence boundaries, each from generated property tables, the rules taken in order, in linear time. They pass all of Unicode 15.1's LineBreakTest (10,274 cases), GraphemeBreakTest (1,187), WordBreakTest (1,826) and SentenceBreakTest (512). --- text/grapheme.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 text/grapheme.c (limited to 'text/grapheme.c') diff --git a/text/grapheme.c b/text/grapheme.c new file mode 100644 index 0000000..b2eb136 --- /dev/null +++ b/text/grapheme.c @@ -0,0 +1,45 @@ +/* Grapheme clusters (UAX #29, Unicode 15.1): text/text.h. One pass, the + * rules in order; three pieces of state for the rules that look further + * back than one character (GB9c, GB11, GB12-13). */ +#include "text.h" + +void vega_graphemes(const uint32_t *codes, size_t count, uint8_t *breaks) { + breaks[0] = count > 0; /* GB1 */ + /* conjunct: 1 after a consonant and extends, 2 once a linker came too + * (GB9c); pictograph: 1 after a pictograph and extends, 2 after its ZWJ + * (GB11); regional: indicators in a row (GB12, GB13). */ + unsigned conjunct = 0, pictograph = 0, regional = 0, before = 0; + for (size_t i = 0; i < count; ++i) { + uint8_t property = vega_grapheme(codes[i]); + unsigned now = property & 15, incb = property >> 4; + bool pictographic = vega_extended_pictographic(codes[i]); + if (i) { + bool join; + if (before == VEGA_GCB_CR && now == VEGA_GCB_LF) join = true; /* GB3 */ + else if (before == VEGA_GCB_CONTROL || before == VEGA_GCB_CR || before == VEGA_GCB_LF) join = false; /* GB4 */ + else if (now == VEGA_GCB_CONTROL || now == VEGA_GCB_CR || now == VEGA_GCB_LF) join = false; /* GB5 */ + else if (before == VEGA_GCB_L && (now == VEGA_GCB_L || now == VEGA_GCB_V || now == VEGA_GCB_LV || now == VEGA_GCB_LVT)) + join = true; /* GB6 */ + else if ((before == VEGA_GCB_LV || before == VEGA_GCB_V) && (now == VEGA_GCB_V || now == VEGA_GCB_T)) + join = true; /* GB7 */ + else if ((before == VEGA_GCB_LVT || before == VEGA_GCB_T) && now == VEGA_GCB_T) join = true; /* GB8 */ + else if (now == VEGA_GCB_EXTEND || now == VEGA_GCB_ZWJ) join = true; /* GB9 */ + else if (now == VEGA_GCB_SPACING_MARK) join = true; /* GB9a */ + else if (before == VEGA_GCB_PREPEND) join = true; /* GB9b */ + else if (conjunct == 2 && incb == VEGA_INCB_CONSONANT) join = true; /* GB9c */ + else if (pictograph == 2 && pictographic) join = true; /* GB11 */ + else if (before == VEGA_GCB_RI && now == VEGA_GCB_RI && regional % 2) join = true; /* GB12, GB13 */ + else join = false; /* GB999 */ + breaks[i] = !join; + } + conjunct = incb == VEGA_INCB_CONSONANT ? 1 + : conjunct && incb == VEGA_INCB_LINKER ? 2 + : conjunct && incb == VEGA_INCB_EXTEND ? conjunct : 0; + pictograph = pictographic ? 1 + : pictograph == 1 && now == VEGA_GCB_EXTEND ? 1 + : pictograph == 1 && now == VEGA_GCB_ZWJ ? 2 : 0; + regional = now == VEGA_GCB_RI ? regional + 1 : 0; + before = now; + } + breaks[count] = count > 0; /* GB2 */ +} -- cgit v1.2.3