Show what would act on a screen as a picture on every surface, and keep a value's bidi within it

Speech wrote a value's C1 controls (U+009B is a CSI to some terminals),
DEL, line separators and bidi overrides as they were; the CLI and the
TUI wrote line separators (a pipe's reader may break its line there) and
the controls that turn text around (a terminal then reverses the rest of
its line, other values' cells too); the web page showed controls as they
were and let a label's override reverse its badge. Now every surface
shows controls, DEL, C1 and line and paragraph separators as pictures
(vega_visible); a terminal's cells show embeddings, overrides and
isolates as U+FFFD too (vega_cell); speech says a space for the one and
nothing for the other; the page shows the pictures and isolates each
value, while /screen's JSON keeps values as they are. screen_meaning
checks the CLI, the TUI, speech and the JSON against that rule, and that
nothing a value holds reaches any output as a control; the browser check
sees the page show it.

5 files changed, 51 insertions, 13 deletions

diff --git a/surfaces/draw.c b/surfaces/draw.c
index ec0b536..3135705 100644
--- a/surfaces/draw.c
+++ b/surfaces/draw.c
@@ -6,15 +6,16 @@ static const char *const tone_names[VEGA_TONES] = {"normal", "muted", "accent",
 
 /* ---- CLI ------------------------------------------------------------ */
 
-/* Text for a pipe: control characters become their pictures so a value never
- * breaks a line. */
+/* Text for a pipe: what would act on the output instead of being shown
+ * becomes its picture (vega_cell), so a value never breaks a line nor turns
+ * one around. */
 static void put_text(FILE *out, const char *text, size_t length) {
     for (size_t at = 0; at < length;) {
         uint32_t codepoint;
         size_t size = vega_decode(text, length, at, &codepoint);
-        if (vega_visible(codepoint) == codepoint) fwrite(text + at, 1, size, out);
+        if (vega_cell(codepoint) == codepoint) fwrite(text + at, 1, size, out);
         else {
-            uint32_t picture = vega_visible(codepoint);
+            uint32_t picture = vega_cell(codepoint);
             fputc(0xE0 | (int)(picture >> 12), out);
             fputc(0x80 | (int)(picture >> 6 & 0x3F), out);
             fputc(0x80 | (int)(picture & 0x3F), out);
@@ -125,10 +126,29 @@ void vega_print(FILE *out, PureApp *app, const VegaScreen *screen, bool color) {
 
 /* ---- NUI ------------------------------------------------------------ */
 
-/* Spoken text: control characters become spaces, and trailing ones go. */
+/* A code point as it is said: what would act on the output (controls, line
+ * and paragraph separators) as a space, the controls that turn text around
+ * not at all (0), else itself. */
+static uint32_t spoken(uint32_t codepoint) {
+    if (vega_visible(codepoint) != codepoint) return ' ';
+    return vega_cell(codepoint) != codepoint ? 0 : codepoint;
+}
+
+/* Spoken text: as spoken says, without the spaces it ends with. */
 static void speak(FILE *out, const char *text, size_t length) {
-    while (length && ((unsigned char)text[length - 1] <= 0x20)) --length;
-    for (size_t i = 0; i < length; ++i) fputc((unsigned char)text[i] < 0x20 ? ' ' : text[i], out);
+    size_t end = 0;
+    for (size_t at = 0; at < length;) {
+        uint32_t codepoint, said;
+        at += vega_decode(text, length, at, &codepoint);
+        if ((said = spoken(codepoint)) && said != ' ') end = at;
+    }
+    for (size_t at = 0; at < end;) {
+        uint32_t codepoint, said;
+        size_t size = vega_decode(text, end, at, &codepoint);
+        if ((said = spoken(codepoint)) == ' ') fputc(' ', out);
+        else if (said) fwrite(text + at, 1, size, out);
+        at += size;
+    }
 }
 
 /* "Item 2 of 3: milk, done." Tones other than normal and muted are said. */
@@ -292,7 +312,7 @@ static void put_cells(Pen *pen, const char *text, size_t length) {
     for (size_t at = 0; at < length;) {
         uint32_t codepoint;
         at += vega_decode(text, length, at, &codepoint);
-        codepoint = vega_visible(codepoint);
+        codepoint = vega_cell(codepoint);
         unsigned width = vega_width(codepoint);
         if (!width) {
             if (pen->column && !line[pen->column - 1].mark) {
diff --git a/surfaces/pixels.c b/surfaces/pixels.c
index d58d5b2..59d5d07 100644
--- a/surfaces/pixels.c
+++ b/surfaces/pixels.c
@@ -312,8 +312,7 @@ static bool set_type(const VegaGrid *grid, VegaFace *face, uint32_t *out, size_t
             for (size_t at = 0; at < t->length;) {
                 uint32_t code;
                 at += vega_decode(grid->text + t->at, t->length, at, &code);
-                code = vega_visible(code);
-                w.codes[count++] = code == 0x2028 || code == 0x2029 ? ' ' : code; /* one line: no breaks in it */
+                w.codes[count++] = vega_visible(code); /* one line: no breaks in it */
             }
         }
         if (j < n && grid->spans[w.order[j]].row == s->row && grid->spans[w.order[j]].column < end)
diff --git a/surfaces/vega.h b/surfaces/vega.h
index 0ab87dd..c172d31 100644
--- a/surfaces/vega.h
+++ b/surfaces/vega.h
@@ -91,8 +91,16 @@ unsigned long vega_evaluations(void);
 unsigned vega_width(uint32_t codepoint);
 /* The code point at text[at] (valid UTF-8) and the bytes it takes. */
 size_t vega_decode(const char *text, size_t length, size_t at, uint32_t *codepoint);
-/* A control character shown as its control picture (U+2400 block), else itself. */
+/* What a value's code point is shown as, on every surface: a control
+ * character as its control picture (the U+2400 block, DEL too), a C1 control
+ * as U+FFFD, a line or paragraph separator as U+2424 (a value never breaks
+ * its line), else itself. */
 uint32_t vega_visible(uint32_t codepoint);
+/* In a terminal's cell (the TUI, the CLI) also the controls that would turn
+ * the rest of the terminal's line around (embeddings, overrides, isolates),
+ * as U+FFFD: pixels lay a value's text out themselves, within its place. */
+uint32_t vega_cell(uint32_t codepoint);
+/* The cells text takes, as vega_cell shows it. */
 unsigned vega_text_width(const char *text, size_t length);
 
 /* CLI: everything, as plain lines for pipes; color only when asked. */
diff --git a/surfaces/web.c b/surfaces/web.c
index a4759c4..dfa9470 100644
--- a/surfaces/web.c
+++ b/surfaces/web.c
@@ -193,6 +193,8 @@ static const char *const page_parts[] = {
 "ul{list-style:none;margin:0;padding:0;border:1px solid var(--line);border-radius:8px;overflow:hidden}"
 "li,tbody tr{cursor:pointer}li{display:flex;gap:10px;padding:7px 12px;border-top:1px solid var(--line)}"
 "li:first-child{border-top:0}.label{flex:1;overflow:hidden;text-overflow:ellipsis;white-space:pre}"
+/* Each value its own paragraph as bidi goes: what turns text around stays in it. */
+".label,.detail,.badge,td,th,dd,dt,h1,h2 span{unicode-bidi:isolate}"
 ".detail{color:var(--muted);white-space:pre}.badge{font-size:12px;padding:1px 8px;border:1px solid currentColor;"
 "border-radius:10px;align-self:center}[aria-selected=true]{background:var(--focus);color:#fff}"
 "table{width:100%;border-collapse:collapse;border:1px solid var(--line)}th{text-align:left;color:var(--muted);"
@@ -229,7 +231,10 @@ static const char *const page_parts[] = {
 "finally{busy=false;if(again){again=false;refresh()}}}"
 "function fit(parent,count,make){while(parent.children.length>count)parent.lastChild.remove();"
 "while(parent.children.length<count)parent.append(make());return[...parent.children]}"
-"function text(node,value){if(node.textContent!==value)node.textContent=value}"
+/* A value shown as the other surfaces show it (vega_visible): controls as their pictures. */
+"function visible(value){return String(value).replace(/[\\x00-\\x1f\\x7f-\\x9f\\u2028\\u2029]/g,c=>{const n=c.charCodeAt(0);"
+"return String.fromCharCode(n<32?0x2400+n:n===127?0x2421:n<160?0xfffd:0x2424)})}"
+"function text(node,value){value=visible(value);if(node.textContent!==value)node.textContent=value}"
 "function attr(node,name,value){if(value==null)node.removeAttribute(name);"
 "else if(node.getAttribute(name)!==String(value))node.setAttribute(name,value)}"
 "const views=new Map();"
diff --git a/surfaces/width.c b/surfaces/width.c
index 1e893f6..645e309 100644
--- a/surfaces/width.c
+++ b/surfaces/width.c
@@ -46,9 +46,15 @@ uint32_t vega_visible(uint32_t codepoint) {
     if (codepoint < 0x20) return 0x2400 + codepoint;
     if (codepoint == 0x7F) return 0x2421;
     if (codepoint >= 0x80 && codepoint < 0xA0) return 0xFFFD;
+    if (codepoint == 0x2028 || codepoint == 0x2029) return 0x2424;
     return codepoint;
 }
 
+uint32_t vega_cell(uint32_t codepoint) {
+    if ((codepoint >= 0x202A && codepoint <= 0x202E) || (codepoint >= 0x2066 && codepoint <= 0x2069)) return 0xFFFD;
+    return vega_visible(codepoint);
+}
+
 size_t vega_decode(const char *text, size_t length, size_t at, uint32_t *codepoint) {
     const unsigned char *p = (const unsigned char *)text + at;
     size_t left = length - at, need = p[0] < 0x80 ? 1 : p[0] < 0xE0 ? 2 : p[0] < 0xF0 ? 3 : 4;
@@ -64,7 +70,7 @@ unsigned vega_text_width(const char *text, size_t length) {
     for (size_t at = 0; at < length;) {
         uint32_t codepoint;
         at += vega_decode(text, length, at, &codepoint);
-        width += vega_width(vega_visible(codepoint));
+        width += vega_width(vega_cell(codepoint));
     }
     return width;
 }