From 0d2e04f1c9d05a598deb443ba067ef61e0b9cab3 Mon Sep 17 00:00:00 2001 From: eunhhu Date: Mon, 28 Sep 2026 12:57:44 +0100 Subject: Break text into lines, graphemes, words and sentences as Unicode's tests require UAX #14 line breaking (LB1-LB31, numbers tailored as Unicode's tests are) and UAX #29's grapheme cluster, word and sentence boundaries, each from generated property tables, the rules taken in order, in linear time. They pass all of Unicode 15.1's LineBreakTest (10,274 cases), GraphemeBreakTest (1,187), WordBreakTest (1,826) and SentenceBreakTest (512). --- text/text.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'text/text.h') diff --git a/text/text.h b/text/text.h index 3704bbd..298a7b4 100644 --- a/text/text.h +++ b/text/text.h @@ -49,6 +49,50 @@ static inline bool vega_mark(uint32_t code) { return c == VEGA_MN || c == VEGA_MC || c == VEGA_ME; } +/* Line_Break (UAX #14), in the order of text/unicode.py. */ +typedef enum { + VEGA_LB_BK, VEGA_LB_CR, VEGA_LB_LF, VEGA_LB_NL, VEGA_LB_SP, VEGA_LB_ZW, VEGA_LB_ZWJ, VEGA_LB_CM, VEGA_LB_WJ, + VEGA_LB_GL, VEGA_LB_BA, VEGA_LB_BB, VEGA_LB_B2, VEGA_LB_HY, VEGA_LB_CB, VEGA_LB_CL, VEGA_LB_CP, VEGA_LB_EX, + VEGA_LB_IN, VEGA_LB_NS, VEGA_LB_OP, VEGA_LB_QU, VEGA_LB_IS, VEGA_LB_NU, VEGA_LB_PO, VEGA_LB_PR, VEGA_LB_SY, + VEGA_LB_AI, VEGA_LB_AK, VEGA_LB_AL, VEGA_LB_AP, VEGA_LB_AS, VEGA_LB_CJ, VEGA_LB_EB, VEGA_LB_EM, VEGA_LB_H2, + VEGA_LB_H3, VEGA_LB_HL, VEGA_LB_ID, VEGA_LB_JL, VEGA_LB_JV, VEGA_LB_JT, VEGA_LB_RI, VEGA_LB_SA, VEGA_LB_SG, + VEGA_LB_VF, VEGA_LB_VI, VEGA_LB_XX, VEGA_LB_CLASSES +} VegaLineBreak; +VegaLineBreak vega_line_break(uint32_t code); +/* Opening or closing punctuation that is East Asian (Fullwidth, Wide or + * Halfwidth): what rule LB30 leaves out. */ +bool vega_east_asian_bracket(uint32_t code); +/* Where a line may break (UAX #14, with the number tailoring of its + * Example 7, as Unicode's tests do): breaks[i] is the boundary before + * codes[i], i from 0 to count (breaks has count + 1 room). */ +enum { VEGA_BREAK_NONE, VEGA_BREAK_ALLOWED, VEGA_BREAK_MANDATORY }; +bool vega_line_breaks(const uint32_t *codes, size_t count, uint8_t *breaks); + +/* Grapheme_Cluster_Break in the low four bits (in the order of + * text/unicode.py), Indic_Conjunct_Break in the high four; and where + * grapheme clusters begin (UAX #29): breaks[i] is 1 before codes[i], i + * from 0 to count. */ +enum { VEGA_GCB_OTHER, VEGA_GCB_CR, VEGA_GCB_LF, VEGA_GCB_CONTROL, VEGA_GCB_EXTEND, VEGA_GCB_ZWJ, VEGA_GCB_RI, + VEGA_GCB_PREPEND, VEGA_GCB_SPACING_MARK, VEGA_GCB_L, VEGA_GCB_V, VEGA_GCB_T, VEGA_GCB_LV, VEGA_GCB_LVT }; +enum { VEGA_INCB_NONE, VEGA_INCB_LINKER, VEGA_INCB_CONSONANT, VEGA_INCB_EXTEND }; +uint8_t vega_grapheme(uint32_t code); +void vega_graphemes(const uint32_t *codes, size_t count, uint8_t *breaks); + +/* Word_Break (in the order of text/unicode.py), and where words begin and + * end (UAX #29): breaks[i] is 1 before codes[i], i from 0 to count. */ +enum { VEGA_WB_OTHER, VEGA_WB_CR, VEGA_WB_LF, VEGA_WB_NEWLINE, VEGA_WB_EXTEND, VEGA_WB_ZWJ, VEGA_WB_RI, VEGA_WB_FORMAT, + VEGA_WB_KATAKANA, VEGA_WB_HEBREW_LETTER, VEGA_WB_ALETTER, VEGA_WB_SINGLE_QUOTE, VEGA_WB_DOUBLE_QUOTE, + VEGA_WB_MIDNUMLET, VEGA_WB_MIDLETTER, VEGA_WB_MIDNUM, VEGA_WB_NUMERIC, VEGA_WB_EXTENDNUMLET, VEGA_WB_WSEGSPACE }; +uint8_t vega_word(uint32_t code); +bool vega_words(const uint32_t *codes, size_t count, uint8_t *breaks); + +/* Sentence_Break (in the order of text/unicode.py), and where sentences + * begin (UAX #29): breaks[i] is 1 before codes[i], i from 0 to count. */ +enum { VEGA_SB_OTHER, VEGA_SB_CR, VEGA_SB_LF, VEGA_SB_EXTEND, VEGA_SB_SEP, VEGA_SB_FORMAT, VEGA_SB_SP, VEGA_SB_LOWER, + VEGA_SB_UPPER, VEGA_SB_OLETTER, VEGA_SB_NUMERIC, VEGA_SB_ATERM, VEGA_SB_SCONTINUE, VEGA_SB_STERM, VEGA_SB_CLOSE }; +uint8_t vega_sentence(uint32_t code); +bool vega_sentences(const uint32_t *codes, size_t count, uint8_t *breaks); + /* Canonical_Combining_Class; a code point's full canonical decomposition * (0 when it has none; Hangul syllables are left to vega_nfd); the primary * composite of two code points, or 0. */ -- cgit v1.2.3