Pure 140's line breaking (UAX #14, Unicode 16.0): examples/linebreak.aup, each boundary of a line of code points decided by the first rule in the standard's order (as Vega's text/linebreak.c), the properties loaded as data (a load message's rows into a stored list of ranges, each its class after LB1, initial and final quotation, East Asian, pictographic and unassigned; found by a binary search by position), each position's values in a record passed to decide (as C's const Text *t), the classes zero-argument fns (fn BK = 0; ...); the boundaries a bytes state and a view; 176 lines against Python's 99

1 files changed, 189 insertions, 0 deletions

diff --git a/examples/linebreak.aup b/examples/linebreak.aup
new file mode 100644
index 0000000..f9735c5
--- /dev/null
+++ b/examples/linebreak.aup
@@ -0,0 +1,189 @@
+# Line breaking (UAX #14, Unicode 16.0): each boundary of a line of code points decided by the first rule, in the
+# standard's order, that matches it (as Vega's text/linebreak.c). The properties come in as data, ranges of code points
+# loaded once: each its class after LB1, whether an initial or a final quotation mark (Pi, Pf), East Asian (width F, W
+# or H), and pictographic and unassigned (Extended_Pictographic and Cn). A line's boundaries, the ends included, are 0
+# (none), 1 (allowed) or 2 (mandatory).
+record Range {
+  start = 0
+  class = 0
+  pi = false
+  pf = false
+  east = false
+  pictographic = false
+}
+record Point {
+  code = 0
+}
+record Text {                                    # each position's: its code, class, range, and what the rules look back at
+  codes: list of number
+  cls: list of number
+  eff: list of number                            # its unit's class: a mark takes its base's (LB9), a lone mark is AL (LB10)
+  joined: list of bool                           # a mark LB9 joins to the character before
+  start: list of number                          # where its unit starts
+  nonspace: list of number                       # where the last unit up to it that is not a space starts, or -1
+  numeric: list of bool                          # its unit ends NU (NU | SY | IS)*
+  ri: list of number                             # how many regional indicators end at its unit
+  at: list of number                             # its range
+}
+state ranges: list of Range                      # by start, from 0
+state breaks: bytes                              # the last line's boundaries
+view boundaries = breaks
+input load = message { rows: list of Range }
+input line = message { points: list of Point }
+
+fn BK = 0; fn CR = 1; fn LF = 2; fn NL = 3; fn SP = 4; fn ZW = 5; fn ZWJ = 6; fn CM = 7; fn WJ = 8; fn GL = 9
+fn BA = 10; fn BB = 11; fn B2 = 12; fn HY = 13; fn CB = 14; fn CL = 15; fn CP = 16; fn EX = 17; fn IN = 18; fn NS = 19
+fn OP = 20; fn QU = 21; fn IS = 22; fn NU = 23; fn PO = 24; fn PR = 25; fn SY = 26; fn AK = 27; fn AL = 28; fn AP = 29
+fn AS = 30; fn EB = 31; fn EM = 32; fn H2 = 33; fn H3 = 34; fn HL = 35; fn ID = 36; fn JL = 37; fn JV = 38; fn JT = 39
+fn RI = 40; fn VF = 41; fn VI = 42
+fn NONE = 0; fn ALLOWED = 1; fn MANDATORY = 2
+fn end(c) = c == BK || c == CR || c == LF || c == NL
+
+on load {
+  for r in load.rows { ranges.append(Range { ..r }) }
+}
+
+# The range holding code point c (ranges by start: the last whose start is c or before).
+fn range(c) {
+  let low, high = 0, ranges.count
+  while high - low > 1 {
+    let middle = (low + high) // 2
+    if ranges[middle].start <= c { low = middle } else { high = middle }
+  }
+  return low
+}
+
+fn next_unit(t, i) {
+  let j = i + 1
+  while j < t.codes.count && t.joined[j] { j = j + 1 }
+  return j
+}
+
+fn aksara(t, u) = t.eff[u] == AK || t.eff[u] == AS || t.codes[u] == 9676   # U+25CC
+fn pi(t, k) = ranges[t.at[k]].pi
+fn pf(t, k) = ranges[t.at[k]].pf
+fn east(t, k) = ranges[t.at[k]].east
+
+fn decide(t, i) {
+  let n = t.codes.count
+  let a, b, A, B = t.cls[i - 1], t.cls[i], t.eff[i - 1], t.eff[i]
+  let unit, before = t.start[i - 1], t.nonspace[i - 1]
+  let base = t.codes[unit]
+  if a == BK { return MANDATORY }                                                     # LB4
+  if a == CR && b == LF { return NONE }                                               # LB5
+  if a == CR || a == LF || a == NL { return MANDATORY }
+  if end(b) { return NONE }                                                           # LB6
+  if b == SP || b == ZW { return NONE }                                               # LB7
+  if before >= 0 && t.eff[before] == ZW { return ALLOWED }                            # LB8
+  if a == ZWJ { return NONE }                                                         # LB8a
+  if t.joined[i] { return NONE }                                                      # LB9
+  if A == WJ || B == WJ { return NONE }                                               # LB11
+  if A == GL { return NONE }                                                          # LB12
+  if B == GL && A != SP && A != BA && A != HY { return NONE }                         # LB12a
+  if B == CL || B == CP || B == EX || B == SY { return NONE }                         # LB13
+  if before >= 0 && t.eff[before] == OP { return NONE }                               # LB14
+  if before >= 0 && t.eff[before] == QU && pi(t, before) {                            # LB15a
+    if before == 0 { return NONE }
+    let q = t.eff[before - 1]
+    if end(q) || q == OP || q == QU || q == GL || q == SP || q == ZW { return NONE }
+  }
+  if B == QU && pf(t, i) {                                                            # LB15b
+    let j = next_unit(t, i)
+    if j == n { return NONE }
+    let e = t.eff[j]
+    if e == SP || e == GL || e == WJ || e == CL || e == QU || e == CP || e == EX || e == IS || e == SY || end(e) || e == ZW {
+      return NONE
+    }
+  }
+  if A == SP && B == IS {                                                             # LB15c
+    let j = next_unit(t, i)
+    if j < n && t.eff[j] == NU { return ALLOWED }
+  }
+  if B == IS { return NONE }                                                          # LB15d
+  if B == NS && before >= 0 && (t.eff[before] == CL || t.eff[before] == CP) { return NONE } # LB16
+  if B == B2 && before >= 0 && t.eff[before] == B2 { return NONE }                    # LB17
+  if A == SP { return ALLOWED }                                                       # LB18
+  if B == QU && !pi(t, i) { return NONE }                                             # LB19
+  if A == QU && !pf(t, unit) { return NONE }
+  if B == QU {                                                                        # LB19a
+    let j = next_unit(t, i)
+    if !east(t, unit) || j == n || !east(t, j) { return NONE }
+  }
+  if A == QU && (!east(t, i) || unit == 0 || !east(t, t.start[unit - 1])) { return NONE }
+  if B == CB || A == CB { return ALLOWED }                                            # LB20
+  if (A == HY || base == 8208) && B == AL {                                           # LB20a (U+2010)
+    let w = if unit > 0 { t.eff[unit - 1] } else { SP }
+    if end(w) || w == SP || w == ZW || w == CB || w == GL { return NONE }
+  }
+  if B == BA || B == HY || B == NS || A == BB { return NONE }                         # LB21
+  if (A == HY || (A == BA && !east(t, unit))) && unit > 0 && t.eff[unit - 1] == HL && B != HL { return NONE } # LB21a
+  if A == SY && B == HL { return NONE }                                               # LB21b
+  if B == IN { return NONE }                                                          # LB22
+  if ((A == AL || A == HL) && B == NU) || (A == NU && (B == AL || B == HL)) { return NONE } # LB23
+  if (A == PR && (B == ID || B == EB || B == EM)) || ((A == ID || A == EB || A == EM) && B == PO) { return NONE } # LB23a
+  if ((A == PR || A == PO) && (B == AL || B == HL)) || ((A == AL || A == HL) && (B == PR || B == PO)) { return NONE } # LB24
+  if (B == PO || B == PR) && (t.numeric[i - 1] || ((A == CL || A == CP) && unit > 0 && t.numeric[unit - 1])) { return NONE } # LB25
+  if A == PO || A == PR {
+    if B == NU { return NONE }
+    if B == OP {
+      let j = next_unit(t, i)
+      if j < n && t.eff[j] == IS { j = next_unit(t, j) }
+      if j < n && t.eff[j] == NU { return NONE }
+    }
+  }
+  if (A == HY || A == IS || t.numeric[i - 1]) && B == NU { return NONE }
+  if A == JL && (B == JL || B == JV || B == H2 || B == H3) { return NONE }            # LB26
+  if ((A == JV || A == H2) && (B == JV || B == JT)) || ((A == JT || A == H3) && B == JT) { return NONE }
+  let jamo = A == JL || A == JV || A == JT || A == H2 || A == H3                       # LB27
+  if (jamo && B == PO) || (A == PR && (B == JL || B == JV || B == JT || B == H2 || B == H3)) { return NONE }
+  if (A == AL || A == HL) && (B == AL || B == HL) { return NONE }                     # LB28
+  if A == AP && aksara(t, i) { return NONE }                                          # LB28a
+  if aksara(t, unit) && (B == VF || B == VI) { return NONE }
+  if A == VI && unit > 0 && aksara(t, t.start[unit - 1]) && (B == AK || t.codes[i] == 9676) { return NONE }
+  if aksara(t, unit) && aksara(t, i) {
+    let j = next_unit(t, i)
+    if j < n && t.eff[j] == VF { return NONE }
+  }
+  if A == IS && (B == AL || B == HL) { return NONE }                                  # LB29
+  if (A == AL || A == HL || A == NU) && B == OP && !east(t, i) { return NONE }        # LB30
+  if A == CP && !east(t, unit) && (B == AL || B == HL || B == NU) { return NONE }
+  if A == RI && B == RI && t.ri[i - 1] % 2 == 1 { return NONE }                       # LB30a
+  if B == EM && (A == EB || ranges[t.at[unit]].pictographic) { return NONE }          # LB30b
+  return ALLOWED                                                                      # LB31
+}
+
+on line {
+  let codes: list of number
+  let cls: list of number
+  let eff: list of number
+  let start: list of number
+  let nonspace: list of number
+  let ri: list of number
+  let at: list of number
+  let joined: list of bool
+  let numeric: list of bool
+  for p in line.points {
+    let i = codes.count
+    let r = range(p.code)
+    let c = ranges[r].class
+    let mark = c == CM || c == ZWJ
+    let previous = if i > 0 { eff[i - 1] } else { SP }
+    let joins = mark && i > 0 && !end(previous) && previous != SP && previous != ZW   # LB9, LB10
+    let e = if joins { previous } else if mark { AL } else { c }
+    codes.append(p.code)
+    cls.append(c)
+    at.append(r)
+    joined.append(joins)
+    eff.append(e)
+    start.append(if joins { start[i - 1] } else { i })
+    nonspace.append(if e != SP { start[i] } else if i > 0 { nonspace[i - 1] } else { -1 })
+    numeric.append(if joins { numeric[i - 1] } else { e == NU || ((e == SY || e == IS) && i > 0 && numeric[i - 1]) })
+    ri.append(if joins { ri[i - 1] } else if e == RI { (if i > 0 { ri[i - 1] } else { 0 }) + 1 } else { 0 })
+  }
+  let t = Text { codes: codes, cls: cls, eff: eff, joined: joined, start: start, nonspace: nonspace, numeric: numeric, ri: ri,
+                 at: at }
+  let out: list of i64 = [0]                                                          # LB2
+  for i in 1 .. codes.count { out.append(i64(decide(t, i))) }
+  if codes.count > 0 { out.append(2) }                                                # LB3
+  breaks = bytes(out)
+}