Group message fields: to: { city: text }, read as ship.to.city

A message's field block may hold groups, nested as deep as the 31
characters of a path allow; the compiler flattens them into fields
named by their path (to.city), which Pure's module format takes, so
groups cost nothing at run time. A member chain on a message names a
field by its path; one that stops at a group, or names a field the
group lacks, is diagnosed, and a field's own member (m.text.bytes)
still reads the field. A module may have 64 message fields.

The JSON server check posts 100 random bodies with groups two deep
(members shuffled at every level, strings with escapes, control bytes,
Korean, emoji and U+2028) to pure-serve, which reads them in place, and
compares the answers and the table with a Python model; a group's fixed
field from the query and ten malformed group bodies are checked too.
Six diagnostics cover groups.

3 files changed, 151 insertions, 21 deletions

diff --git a/docs/design/001-language-v0.md b/docs/design/001-language-v0.md
index 660abae..7a5b0d6 100644
--- a/docs/design/001-language-v0.md
+++ b/docs/design/001-language-v0.md
@@ -115,6 +115,10 @@ on edit {
 - 필드 타입은 `u32`, `number`(유한한 수), `bytes`, `text`(UTF-8)이고 가변 필드(bytes, text)는 몇 개든 어디에나 둘 수 있다.
   `input add = message { title: text, points: number }`. HTTP 호스트(`pure-serve`)는 JSON 본문의 객체를 이 필드들로
   옮긴다. 도착한 메시지의 text가 UTF-8이 아니거나 수가 유한하지 않으면 그 이벤트는 실패한다.
+- 필드를 그룹으로 묶을 수 있다: `to: { city: text, zip: u32 }`. 그룹 안의 필드는 `메시지.그룹.필드`(`ship.to.city`)로
+  읽고, JSON 본문에서는 안의 객체(`{"to": {"city": "Seoul", "zip": 4}}`), 쿼리에서는 경로(`?to.zip=4`)다. 그룹은 겹칠 수
+  있고, 경로는 31자 이하, 한 메시지의 필드는 그룹 안까지 합쳐 64개 이하다. 모듈에서는 `to.city` 같은 이름의 필드로 펼쳐지므로
+  실행 비용이 없다.
 - `이름[key]`는 그 메시지 키의 문서다. 읽으면 bytes, `.bytes`는 크기(없으면 0)다. 없는 문서를 읽으면 `not_found`.
 - `replace(a .. b, 값)`는 문서의 `[a, b)`를 바꾼다. 정수가 아니거나 범위 밖이면 이벤트가 `limit`으로 실패한다.
 - `require 조건`이 거짓이면 이벤트가 `limit`으로 실패한다. 크기 정책처럼 저장소의 특수 인자였던 것이 문장 하나가 된다.
diff --git a/src/compile.c b/src/compile.c
index 27176a1..bccc2c3 100644
--- a/src/compile.c
+++ b/src/compile.c
@@ -9,7 +9,7 @@
 
 /* Limits and opcodes of Pure module format v5. */
 enum { STATES = 32, INPUTS = 32, VIEWS = 64, HANDLERS = 64, CODE = 65535, STACK = 64, NAME = 32, JOINS = 8,
-       DEPTH = 64, FIELDS = 32, COLLECTIONS = 1, CONSTANTS = 4096, CONSTANT_BYTES = 1 << 20,
+       DEPTH = 64, FIELDS = 64, COLLECTIONS = 1, CONSTANTS = 4096, CONSTANT_BYTES = 1 << 20,
        RECORDS = 8, RECORD_FIELDS = 64, FIELDS_PER_RECORD = 16, FNS = 64, PARAMS = 8, MODULES = 9 };
 enum { CONST, LOAD, DT, ADD, SUB, MUL, DIV, NEG, LT, LE, GT, GE, EQ, NE, AND, OR, NOT,
        MIN, MAX, CLAMP, STORE, JZ, JMP, FIELD, DOC, DOC_SIZE, LEN, REPLACE, REPLY, REQUIRE,
@@ -968,19 +968,42 @@ static bool key_name(const char *key) {
     return false;
 }
 
+/* Whether a message already has a field or a group of this path. */
+static bool path_taken(const Input *in, const char *path) {
+    size_t length = strlen(path);
+    for (unsigned i = 0; i < in->fields; ++i) {
+        const char *name = in->field_names[i];
+        size_t size = strlen(name);
+        if (!strcmp(name, path) || (size > length && !memcmp(name, path, length) && name[length] == '.') ||
+            (length > size && !memcmp(path, name, size) && path[size] == '.')) return true;
+    }
+    return false;
+}
+
 /* { name: type ... }: u32 and number are fixed, bytes and text of any length
- * (Pure lays them out: docs/design/008-module-format.md). */
-static void message_fields(Compiler *c, Input *in) {
+ * (Pure lays them out: docs/design/008-module-format.md). name: { ... } is a
+ * group of fields inside the message, named group.name (a JSON object inside
+ * a request's body). */
+static void message_fields(Compiler *c, Input *in, const char *group) {
     if (!is(c, "{")) return; /* the field block opens on the declaration's line */
     take(c);
     separators(c);
+    unsigned first = in->fields;
     while (!is(c, "}")) {
         Token name = expect_name(c, "a field name");
         if (reserved_name(name.text)) fail_at(c, name.line, name.column, "'%s' is reserved", name.text);
-        for (unsigned i = 0; i < in->fields; ++i)
-            if (!strcmp(in->field_names[i], name.text)) fail_at(c, name.line, name.column, "duplicate field '%s'", name.text);
-        if (c->field_count == FIELDS) fail_at(c, name.line, name.column, "at most 32 message fields are allowed");
+        char path[NAME];
+        int length = snprintf(path, sizeof(path), "%s%s%s", group, *group ? "." : "", name.text);
+        if (length < 0 || length >= NAME)
+            fail_at(c, name.line, name.column, "'%s.%s' is longer than %d characters", group, name.text, NAME - 1);
+        if (path_taken(in, path)) fail_at(c, name.line, name.column, "duplicate field '%s'", path);
         expect(c, ":", "':'");
+        if (is(c, "{")) {
+            message_fields(c, in, path);
+            end(c);
+            continue;
+        }
+        if (c->field_count == FIELDS) fail_at(c, name.line, name.column, "at most %d message fields are allowed", FIELDS);
         Token type = expect_name(c, "u32, number, bytes or text");
         static const struct { const char *name; int type; } types[] = {
             {"u32", FIELD_U32}, {"number", FIELD_NUMBER}, {"bytes", FIELD_BYTES}, {"text", FIELD_TEXT}};
@@ -988,14 +1011,16 @@ static void message_fields(Compiler *c, Input *in) {
         for (size_t t = 0; t < sizeof(types) / sizeof(*types); ++t)
             if (!strcmp(type.text, types[t].name)) field_type = types[t].type;
         if (!field_type) fail_at(c, type.line, type.column, "field types are u32, number, bytes and text");
-        memcpy(in->field_names[in->fields], name.text, NAME);
+        memcpy(in->field_names[in->fields], path, NAME);
         in->field_types[in->fields++] = field_type;
         ++c->field_count;
         end(c);
     }
+    if (*group && in->fields == first) fail_at(c, c->token.line, c->token.column, "the group '%s' has no fields", group);
     take(c);
 }
 
+
 /* A signed number literal: a numeric state or field default. */
 static double signed_number(Compiler *c, const char *what) {
     double sign = 1;
@@ -1273,7 +1298,7 @@ static void declaration(Compiler *c) {
             in->kind = KIND_MESSAGE;
             if (word(c, "by")) { take(c); expect_word(c, "key"); in->keyed = true; }
             ++c->input_count;
-            message_fields(c, in);
+            message_fields(c, in, "");
         } else if (word(c, "text")) {
             take(c);
             in->kind = KIND_TEXT;
@@ -1508,6 +1533,28 @@ static int parameter(const Compiler *c, const char *name) {
     return -1;
 }
 
+/* A member chain on a message, as order.customer.name: the input and the
+ * path after it (customer.name). False for anything else. */
+static bool message_path(Compiler *c, const Expr *e, int *input, char path[NAME]) {
+    if (e->kind != E_MEMBER) return false;
+    const Expr *base = e->args[0];
+    if (base->kind == E_NAME) {
+        if (parameter(c, base->name) >= 0 || c->module) return false;
+        *input = input_index(c, base->name);
+        if (*input < 0 || c->inputs[*input].kind != KIND_MESSAGE) return false;
+        snprintf(path, NAME, "%s", e->member);
+        return true;
+    }
+    char inner[NAME];
+    if (!message_path(c, base, input, inner)) return false;
+    const Input *in = &c->inputs[*input];
+    for (unsigned i = 0; i < in->fields; ++i)
+        if (!strcmp(in->field_names[i], inner)) return false; /* a field's own member, as m.text.bytes */
+    int length = snprintf(path, NAME, "%s.%s", inner, e->member);
+    if (length < 0 || length >= NAME) snprintf(path, NAME, "%s", inner); /* too long: names nothing */
+    return true;
+}
+
 /* A name of the app (a state, input or collection): not visible in a library
  * fn, and hidden by a parameter of the same name. */
 static void app_name(Compiler *c, const Expr *e, const char *name) {
@@ -1588,20 +1635,27 @@ static Kind value(Compiler *c, const Expr *e, bool handler, int height) {
         const Expr *base = e->args[0];
         int library = base->kind == E_NAME && parameter(c, base->name) < 0 ? used_library(c, c->module, base->name) : -1;
         if (library >= 0) return inline_fn(c, e, library, e->member, handler, height); /* library.constant */
+        int message;
+        char path[NAME];
+        if (message_path(c, e, &message, path)) {
+            const Input *in = &c->inputs[message];
+            if (message != c->message)
+                fail_at(c, e->line, e->column, "fields of '%s' are only available in its handler", in->name);
+            size_t length = strlen(path);
+            for (unsigned i = 0; i < in->fields; ++i)
+                if (!strcmp(in->field_names[i], path)) {
+                    emit(c, e->line, e->column, FIELD, i, 0);
+                    int type = in->field_types[i];
+                    return (Kind){type == FIELD_TEXT ? TEXT : type == FIELD_BYTES ? BYTES : NUM, -1};
+                }
+            for (unsigned i = 0; i < in->fields; ++i)
+                if (!strncmp(in->field_names[i], path, length) && in->field_names[i][length] == '.')
+                    fail_at(c, e->line, e->column, "'%s.%s' is a group of fields; read one of them, as in %s.%s",
+                            in->name, path, in->name, in->field_names[i]);
+            fail_at(c, e->line, e->column, "message '%s' has no field '%s'", in->name, path);
+        }
         if (base->kind == E_NAME && parameter(c, base->name) < 0 && !c->module) {
             int input = input_index(c, base->name);
-            if (input >= 0 && c->inputs[input].kind == KIND_MESSAGE) {
-                if (input != c->message)
-                    fail_at(c, e->line, e->column, "fields of '%s' are only available in its handler", base->name);
-                const Input *in = &c->inputs[input];
-                for (unsigned i = 0; i < in->fields; ++i)
-                    if (!strcmp(in->field_names[i], e->member)) {
-                        emit(c, e->line, e->column, FIELD, i, 0);
-                        int type = in->field_types[i];
-                        return (Kind){type == FIELD_TEXT ? TEXT : type == FIELD_BYTES ? BYTES : NUM, -1};
-                    }
-                fail_at(c, e->line, e->column, "message '%s' has no field '%s'", base->name, e->member);
-            }
             if (input >= 0 && c->inputs[input].kind == KIND_KEY)
                 fail_at(c, base->line, base->column, "'%s' is a key input and has no fields", base->name);
             int state = state_index(c, base->name);
diff --git a/tests/check.py b/tests/check.py
index 7e23d94..dcbda5e 100644
--- a/tests/check.py
+++ b/tests/check.py
@@ -89,6 +89,14 @@ DIAGNOSTICS = [
     ('input m = message by key { a: u32 }\nview v = m.a\n', '2:10', "fields of 'm' are only available in its handler"),
     ('state x = 0\ninput m = message by key { a: bytes }\non m {\n  x = m.a\n}\n', '4:7', 'expected a number, not bytes'),
     ('input m = message by key { a: u32 }\non m {\n  reply m.z\n}\n', '3:9', "message 'm' has no field 'z'"),
+    ('input m = message {\n  g: {\n    a: text\n  }\n}\non m {\n  reply m.g\n}\n', '7:9',
+     "'m.g' is a group of fields; read one of them, as in m.g.a"),
+    ('input m = message {\n  g: {\n    a: text\n  }\n}\non m {\n  reply m.g.b\n}\n', '7:9', "message 'm' has no field 'g.b'"),
+    ('input m = message {\n  g: text\n  g: {\n    a: text\n  }\n}\n', '3:3', "duplicate field 'g'"),
+    ('input m = message {\n  g: {\n    a: text\n  }\n  g: {\n    b: text\n  }\n}\n', '5:3', "duplicate field 'g'"),
+    ('input m = message {\n  g: {\n  }\n}\n', '3:3', "the group 'g' has no fields"),
+    ('input m = message {\n  group_of_things: {\n    another_long_name: text\n  }\n}\n', '3:5',
+     "'group_of_things.another_long_name' is longer than 31 characters"),
     ('state x = 0\non frame {\n  require x.bytes\n}\n', '3:11', "a number has no 'bytes'"),
     ('state d: bytes by key\nstate e: bytes by key\n', '2:7', 'at most one keyed collection is allowed'),
     ('state d: bytes by key\ninput m = message by key { a: bytes }\non m {\n  d[key].replace(0 .. m.a, m.a)\n}\n', '4:23',
@@ -1156,17 +1164,44 @@ input add = message {
 input count = message {
   n: u32
 }
+record Shipment {
+  title: text
+  city: text
+  note: text
+  zip = 0
+}
+state shipped: list of Shipment
+input ship = message {
+  title: text
+  to: {
+    city: text
+    zip: u32
+    door: {
+      note: text
+    }
+  }
+}
 on add {
   notes.add(Note { title: add.title, body: add.body, points: add.points })
   reply add.title
 }
 on count { counted = counted + count.n }
+on ship {
+  shipped.append(Shipment { title: ship.title, city: ship.to.city, note: ship.to.door.note, zip: ship.to.zip })
+  reply ship.to.door.note
+}
 view counted_view = counted
 view all = each n in notes {
   title = n.title
   body = n.body
   points = n.points
 }
+view shipments = each s in shipped {
+  title = s.title
+  city = s.city
+  note = s.note
+  zip = s.zip
+}
 """
 
 
@@ -1240,9 +1275,46 @@ def server_json(aup, pure, work, rounds=300):
         assert post('/inputs/add', b'plain', json_body=False)[:2] == (400, 'json')  # two text fields need JSON
         assert post('/inputs/count', b'{"n": -1}')[:2] == (400, 'json_type')
         assert post('/inputs/count', b'{"n": 4294967296}')[:2] == (400, 'json_type')
+        # Groups: objects inside the body, members in any order, strings
+        # read where they are (escaped ones unescaped over themselves).
+        shipments = []
+        for _ in range(rounds // 3):
+            title, city, note = (''.join(rng.choice(pieces) for _ in range(rng.randint(0, 5))) for _ in range(3))
+            zip_code = rng.randint(0, 99999)
+            door = [('note', note)]
+            to = [('city', city), ('zip', zip_code), ('door', dict(door))]
+            rng.shuffle(to)
+            members = [('title', title), ('to', dict(to))]
+            rng.shuffle(members)
+            text = json.dumps(dict(members), ensure_ascii=rng.random() < 0.5)
+            got = post('/inputs/ship', text.encode())
+            assert got == (200, 'none', note.encode()), (text, got)
+            shipments.append({'title': title, 'city': city, 'note': note, 'zip': zip_code})
+        # A group's fixed field from the query, by its path.
+        assert post('/inputs/ship?to.zip=7', b'{"title": "q", "to": {"city": "c", "door": {"note": "n"}}}')[:2] == \
+            (200, 'none')
+        shipments.append({'title': 'q', 'city': 'c', 'note': 'n', 'zip': 7})
+        rows = json.loads(http(host.port, 'GET', f'/views/shipments?count={len(shipments)}')[3])['items']
+        assert rows == shipments, [(a, b) for a, b in zip(rows, shipments) if a != b][:2]
+        grouped = '{"title": "t", "to": {"city": "c", "zip": 1, "door": {"note": "n"}}}'
+        wrong_groups = [
+            (grouped.replace('"zip": 1', '"zip": 1, "lat": 2'), 'json_field'),         # unknown member of a group
+            (grouped.replace('"note": "n"', '"note": "n", "bell": {"x": 1}'), 'json_field'),
+            (grouped.replace('"door": {"note": "n"}', '"door": "n"'), 'json_type'),    # a group as a value
+            (grouped.replace('"title": "t"', '"title": {"a": "t"}'), 'json_type'),    # a field as a group
+            (grouped.replace('"door": {"note": "n"}', '"door": {}'), 'missing_field'),
+            (grouped.replace('"zip": 1', '"zip": 1, "zip": 2'), 'json_field'),
+            (grouped.replace('"zip": 1', '"zip": [1]'), 'json_type'),
+            (grouped.replace('"zip": 1', '"zip": null'), 'json_type'),
+            (grouped.replace('"note": "n"', '"note": "n\\ud800"'), 'json'),
+            (grouped.replace('"door": {"note": "n"}}', '"door": {"note": "n"}'), 'json'),  # unclosed
+        ]
+        for body, error in wrong_groups:
+            assert post('/inputs/ship', body.encode())[:2] == (400, error), (body, post('/inputs/ship', body.encode()))
+        assert post('/inputs/ship?to.zip=7', grouped.encode())[:2] == (400, 'json_field')  # given twice
     finally:
         host.stop()
-    return rounds + len(wrong)
+    return rounds + len(wrong) + rounds // 3 + len(wrong_groups)
 
 
 BEAT_SOURCE = """state beats = 0