027's 4f-3b: no fixed count of records or of their fields in all: the header's u16s alone (16 and 128 before); a stored list's record graph (PureLayout) holds a few records itself and more on the heap, 256 at most (an image numbers them in a byte), an image or a checkpoint not written then without room for it; a record's 16 fields kept, the list engine's width (a leaf's columns, a run's columns and narrow cells, the arrays' and packs' columns), to be widened with the lists' shapes sized to their fields; unit tests: 20 records of 10 fields (200 in all), an item of the last appended; a stored list whose six records each hold a list of the next saved and read back after a restart (its graph on the heap); counts: b41 against b40 with batches BFS +0.12%, without BFS -0.09%, the rest within 0.001%; the bank's queries (s40 and s41 in turns, six each) within 0.09% (the first three's in_credit +1.16% its spread: 2.75% between runs of one build); 008 and 027

5 files changed, 125 insertions, 22 deletions

diff --git a/core/module-save.c b/core/module-save.c
index 19e9684..1055b4f 100644
--- a/core/module-save.c
+++ b/core/module-save.c
@@ -198,9 +198,14 @@ static unsigned char *layout_fields(const PureLayout *l, unsigned at, unsigned c
     return out;
 }
 
+/* 0 without room for the record's graph. */
 static size_t image_header(const PureModule *m, uint32_t record, uint64_t seq, unsigned char *out) {
     PureLayout l;
-    pure_layout_of(m, record, &l);
+    pure_layout_begin(&l);
+    if (!pure_layout_of(m, record, &l)) {
+        pure_layout_done(&l);
+        return 0;
+    }
     memcpy(out, image_magic, 4);
     out[4] = (unsigned char)l.record[0].count;
     unsigned char *at = layout_fields(&l, 0, out + 5);
@@ -210,6 +215,7 @@ static size_t image_header(const PureModule *m, uint32_t record, uint64_t seq, u
         *at++ = (unsigned char)l.record[k].count;
         at = layout_fields(&l, k, at);
     }
+    pure_layout_done(&l);
     return (size_t)(put_le(at, seq, 8) - out);
 }
 
@@ -485,7 +491,8 @@ PureResult pure_save_image(PureApp *app, const PureModule *m, uint32_t state, co
     unsigned char *header = malloc(header_room(m));
     if (!header) return PURE_ALLOCATION;
     size_t header_size = image_header(m, m->state_ref[state], app->seq, header);
-    PureResult result = write_image(directory_of(app), m->state_names[state], header, header_size, list);
+    PureResult result = header_size ? write_image(directory_of(app), m->state_names[state], header, header_size, list)
+                                    : PURE_ALLOCATION;
     free(header);
     return result;
 }
@@ -511,7 +518,9 @@ PURE_COLD PureResult pure_save_open_list(PureApp *app, const PureModule *m, uint
     }
     /* The stored record graph: its fields' kinds (PURE_CELL_...: a record's, its lists' PURE_CELL_LIST) and names, and
      * the records of the lists in its fields (an image's alone, 027's stage 3). */
-    PureLayout stored_layout = {.records = 1};
+    PureLayout stored_layout, layout;
+    pure_layout_begin(&stored_layout), pure_layout_begin(&layout);
+    stored_layout.records = 1;
     unsigned count = 0;
     size_t at = 0, items = 0;
     uint64_t holds = 0;
@@ -523,7 +532,7 @@ PURE_COLD PureResult pure_save_open_list(PureApp *app, const PureModule *m, uint
         at = 5;
         ok = ok && read_fields(bytes, size, &at, count, &stored_layout, 0, 256, &lists) && (image || !lists);
         if (ok && lists) { /* the graph's other records */
-            ok = at < size && bytes[at] < PURE_MODULE_RECORDS;
+            ok = at < size && pure_layout_room(&stored_layout, 1u + bytes[at]);
             stored_layout.records = ok ? 1u + bytes[at++] : 1;
             for (unsigned k = 1; ok && k < stored_layout.records; ++k)
                 ok = at < size && bytes[at] <= PURE_RECORD_FIELDS &&
@@ -564,10 +573,10 @@ PURE_COLD PureResult pure_save_open_list(PureApp *app, const PureModule *m, uint
             saved->logged += change->records.size;
         }
     }
-    PureLayout layout;
-    pure_layout_of(m, record, &layout);
-    bool same = stored && pure_layout_same(&stored_layout, &layout);
+    if (result == PURE_OK && !pure_layout_of(m, record, &layout)) result = PURE_ALLOCATION;
+    bool same = stored && result == PURE_OK && pure_layout_same(&stored_layout, &layout);
     if (result == PURE_OK && same && image) {
+        pure_layout_done(&stored_layout), pure_layout_done(&layout);
         *out = from;
         saved->image = holds;
         return PURE_OK;
@@ -587,6 +596,7 @@ PURE_COLD PureResult pure_save_open_list(PureApp *app, const PureModule *m, uint
         }
     }
     pure_list_free(from);
+    pure_layout_done(&stored_layout), pure_layout_done(&layout);
     if (result != PURE_OK) {
         pure_list_free(*out);
         pure_draft_close(*legacy);
@@ -718,6 +728,11 @@ PureCheckpoint *pure_app_checkpoint(PureApp *app) {
     c->header_size = pick == PURE_LIST_OUTBOX ? outbox_header(m, c->seq, c->header) :
                      pick >= PURE_LIST_HEARD ? runtime_header(pick, c->seq, c->header) :
                                                image_header(m, m->state_ref[pick], c->seq, c->header);
+    if (!c->header_size) { /* no room for its record's graph: none now */
+        pure_snapshot_release(c->snapshot);
+        free(c->directory), free(c->header), free(c);
+        return NULL;
+    }
     c->logged = app->logged[slot];
     c->since = app->logged_since[slot];
     app->checkpoint = c;
diff --git a/core/module-save.h b/core/module-save.h
index 4ec50a2..44d3b95 100644
--- a/core/module-save.h
+++ b/core/module-save.h
@@ -79,15 +79,25 @@ uint8_t pure_record_kind(const PureModule *, uint32_t field);
  * order its list fields first meet them, each its fields' kinds (PURE_CELL_...) and names, and a list field's items'
  * record by its number here. A module's graph knows each record's number in the module too. */
 typedef struct {
-    unsigned records;
-    struct {
-        unsigned count;
-        uint8_t kinds[PURE_RECORD_FIELDS], items[PURE_RECORD_FIELDS];
-        char names[PURE_RECORD_FIELDS][PURE_MODULE_NAME];
-    } record[PURE_MODULE_RECORDS];
-    uint16_t module_record[PURE_MODULE_RECORDS];
+    unsigned count;
+    uint8_t kinds[PURE_RECORD_FIELDS], items[PURE_RECORD_FIELDS];
+    char names[PURE_RECORD_FIELDS][PURE_MODULE_NAME];
+} PureLayoutRecord;
+/* Its records in record (a few in few, more on the heap: room of them), at most 256 (a store numbers them in a byte),
+ * module_record beside them. Begun (pure_layout_begin) before use, done (pure_layout_done) after; never copied. */
+typedef struct {
+    unsigned records, room;
+    PureLayoutRecord *record;
+    uint16_t *module_record;
+    PureLayoutRecord few[4];
+    uint16_t few_records[4];
 } PureLayout;
-void pure_layout_of(const PureModule *, uint32_t record, PureLayout *out);
+enum { PURE_LAYOUT_RECORDS = 256 };
+void pure_layout_begin(PureLayout *);
+bool pure_layout_room(PureLayout *, unsigned records); /* false without room, or past PURE_LAYOUT_RECORDS */
+void pure_layout_done(PureLayout *);
+/* false: no room, or a graph of more than PURE_LAYOUT_RECORDS records. */
+bool pure_layout_of(const PureModule *, uint32_t record, PureLayout *out);
 bool pure_layout_same(const PureLayout *, const PureLayout *);
 /* A list with record's fields, taking from `from` (laid out as from_layout's record 0) each field of the same name and
  * kind, or a number, a bool and an i64 from each other when the value fits (pure_cell_as), and giving the others their
diff --git a/core/module.c b/core/module.c
index ba44a85..3d38966 100644
--- a/core/module.c
+++ b/core/module.c
@@ -1712,7 +1712,6 @@ PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule
     out->state_words = (uint16_t)((out->states + 63) / 64 + !out->states);
     if (out->code_count > PURE_MODULE_CODE || strings > PURE_MODULE_STRINGS ||
         out->constants > PURE_MODULE_CONSTANTS || constant_bytes > PURE_MODULE_CONSTANT_BYTES ||
-        out->records > PURE_MODULE_RECORDS || out->record_fields > PURE_MODULE_RECORD_FIELDS ||
         out->functions > PURE_MODULE_FUNCTIONS) {
         result = PURE_LIMIT;
         REFUSE("module exceeds a limit", 8);
@@ -5998,7 +5997,33 @@ static uint8_t cell_kind(const PureModule *m, uint32_t field, bool kept) {
 /* Whether a stored kind's cell is bytes in the stored form: a text's, or a list's. */
 static bool stored_bytes(uint8_t kind) { return pure_cell_text(kind) || kind == PURE_CELL_LIST; }
 
-PURE_COLD void pure_layout_of(const PureModule *m, uint32_t record, PureLayout *out) {
+void pure_layout_begin(PureLayout *l) {
+    l->records = 0, l->room = 4;
+    l->record = l->few, l->module_record = l->few_records;
+}
+
+bool pure_layout_room(PureLayout *l, unsigned records) {
+    if (records <= l->room) return true;
+    if (records > PURE_LAYOUT_RECORDS) return false;
+    PureLayoutRecord *record = malloc(PURE_LAYOUT_RECORDS * sizeof(*record));
+    uint16_t *module_record = malloc(PURE_LAYOUT_RECORDS * sizeof(*module_record));
+    if (!record || !module_record) {
+        free(record), free(module_record);
+        return false;
+    }
+    memcpy(record, l->record, l->records * sizeof(*record));
+    memcpy(module_record, l->module_record, l->records * sizeof(*module_record));
+    pure_layout_done(l);
+    l->record = record, l->module_record = module_record, l->room = PURE_LAYOUT_RECORDS;
+    return true;
+}
+
+void pure_layout_done(PureLayout *l) {
+    if (l->record != l->few) free(l->record), free(l->module_record);
+    l->record = l->few, l->module_record = l->few_records, l->room = 4;
+}
+
+PURE_COLD bool pure_layout_of(const PureModule *m, uint32_t record, PureLayout *out) {
     out->records = 1;
     out->module_record[0] = (uint16_t)record;
     for (unsigned at = 0; at < out->records; ++at) { /* breadth first: each record once, numbered as first met */
@@ -6013,10 +6038,12 @@ PURE_COLD void pure_layout_of(const PureModule *m, uint32_t record, PureLayout *
             uint32_t items = nested_record(m, field);
             unsigned k = 0;
             while (k < out->records && out->module_record[k] != items) ++k;
+            if (k == out->records && !pure_layout_room(out, out->records + 1)) return false;
             if (k == out->records) out->module_record[out->records++] = (uint16_t)items;
             out->record[at].items[f] = (uint8_t)k;
         }
     }
+    return true;
 }
 
 PURE_COLD bool pure_layout_same(const PureLayout *a, const PureLayout *b) {
@@ -6161,10 +6188,11 @@ PURE_COLD PureResult pure_list_migrate(PureList **out, const PureList *from, con
 PURE_COLD static PureResult keep_list(PureList **out, PureList *from, const PureModule *old, uint32_t old_record,
                             const PureModule *m, uint32_t record, bool kept) {
     PureLayout was, now;
-    pure_layout_of(old, old_record, &was);
-    pure_layout_of(m, record, &now);
-    if (pure_layout_same(&was, &now)) { *out = from; return PURE_OK; }
-    return pure_list_migrate(out, from, &was, m, record, kept);
+    pure_layout_begin(&was), pure_layout_begin(&now);
+    PureResult result = !pure_layout_of(old, old_record, &was) || !pure_layout_of(m, record, &now) ? PURE_ALLOCATION :
+                        pure_layout_same(&was, &now) ? (*out = from, PURE_OK) : pure_list_migrate(out, from, &was, m, record, kept);
+    pure_layout_done(&was), pure_layout_done(&now);
+    return result;
 }
 
 /* Open a module in an app. States keep their values by name and kind (texts
diff --git a/core/module.h b/core/module.h
index 8de3e33..26259fe 100644
--- a/core/module.h
+++ b/core/module.h
@@ -18,7 +18,8 @@ enum {
     PURE_MODULE_NAME = 32, PURE_MODULE_STRINGS = 32768, PURE_MODULE_HEADER = 38,
     /* A message's fields at most: a send pushes them all (the stack's), a width of the stored sends too (027's 4f-3a). */
     PURE_MESSAGE_FIELDS = PURE_MODULE_STACK, PURE_MODULE_CONSTANTS = 4096,
-    PURE_MODULE_CONSTANT_BYTES = 1 << 20, PURE_MODULE_RECORDS = 16, PURE_MODULE_RECORD_FIELDS = 128,
+    PURE_MODULE_CONSTANT_BYTES = 1 << 20,
+    /* A record's fields at most: a list's leaf keeps 16 columns (module-list.c, its runs and arrays too). */
     PURE_RECORD_FIELDS = 16, PURE_MODULE_JOINS = 8
 };
 
diff --git a/core/test-module.c b/core/test-module.c
index 4eabdfc..5bd1c8b 100644
--- a/core/test-module.c
+++ b/core/test-module.c
@@ -489,6 +489,8 @@ static void faults(void) {
 
 typedef struct { const char *reason; Spec spec; PureResult result; } Rejection;
 
+static char *new_store(void);
+static void remove_store(const char *path);
 static void rejections(void) {
     static const S s1[] = {{"a", 0}};
     static const S dup[] = {{"a", 0}, {"a", 1}};
@@ -590,6 +592,53 @@ static void rejections(void) {
     double shown;
     CHECK(pure_app_view(&wide, VIEWS - 1, &shown) && shown == 1);
     pure_app_stop(&wide);
+    /* Nor of records or their fields in all (027's 4f-3b): 20 records of 10 fields, an item of the last appended. */
+    enum { RECORDS = 20, EACH = 10 };
+    static RF record_fields[RECORDS][EACH];
+    static R records[RECORDS];
+    static char record_names[RECORDS][8], field_names[EACH][8];
+    for (unsigned f = 0; f < EACH; ++f) snprintf(field_names[f], 8, "f%u", f);
+    for (unsigned r = 0; r < RECORDS; ++r) {
+        snprintf(record_names[r], 8, "R%u", r);
+        for (unsigned f = 0; f < EACH; ++f) record_fields[r][f] = (RF){field_names[f], 0, 0};
+        records[r] = (R){record_names[r], record_fields[r], EACH};
+    }
+    static const T last_list[] = {{"items", 3, RECORDS - 1}};
+    static O appended[EACH + 3];
+    for (unsigned f = 0; f < EACH; ++f) appended[f] = (O){C, 0, f};
+    appended[EACH] = (O){PURE_OP_LIST_APPEND, 1, 0};
+    appended[EACH + 1] = (O){PURE_OP_LIST_COUNT, 1, 0};
+    appended[EACH + 2] = (O){ST, 0, 0};
+    static const H appending[] = {{1, 0, EACH + 3}};
+    CHECK(load((Spec){s1, 1, key_in, 1, NULL, 0, appending, 1, appended, EACH + 3, NULL, 0, last_list, 1, NULL, 0, NULL, 0,
+                      records, RECORDS, NULL, 0}, &module_a) == PURE_OK && module_a.records == RECORDS &&
+          module_a.record_fields == RECORDS * EACH);
+    CHECK(pure_app_start(&wide, &module_a, (PureData){0}) == PURE_OK && press(&wide, 0) == PURE_OK && wide.values[0].num == 1 &&
+          pure_list_get(wide.lists[1], 0, EACH - 1).num == EACH - 1);
+    pure_app_stop(&wide);
+    /* A stored list's record graph past the few a layout holds itself: six records, each a list of the next. */
+    enum { DEEP = 6 };
+    static RF chain_fields[DEEP][2];
+    static R chain[DEEP];
+    for (unsigned r = 0; r < DEEP; ++r) {
+        chain_fields[r][0] = (RF){"n", 0, 0};
+        chain_fields[r][1] = (RF){"next", 4, r + 1 < DEEP ? r + 1 : r}; /* the last a list of itself */
+        chain[r] = (R){record_names[r], chain_fields[r], 2};
+    }
+    static const T chained[] = {{"items", 3, 0}, {"let.none", 3 | (uint32_t)PURE_LIST_ROWS << 8, 1}};
+    static const O add_one[] = {{C, 0, 7}, {PURE_OP_LIST_VALUE, 2, 0}, {PURE_OP_LIST_APPEND, 1, 0}, {PURE_OP_LIST_COUNT, 1, 0},
+                                {ST, 0, 0}};
+    static const H adding[] = {{1, 0, 5}};
+    Spec deep = {s1, 1, key_in, 1, NULL, 0, adding, 1, add_one, 5, NULL, 0, chained, 2, NULL, 0, NULL, 0, chain, DEEP, NULL, 0};
+    CHECK(load(deep, &module_a) == PURE_OK);
+    char *deep_path = new_store();
+    CHECK(pure_app_start(&wide, &module_a, pure_tree_data(deep_path)) == PURE_OK && press(&wide, 0) == PURE_OK &&
+          press(&wide, 0) == PURE_OK && pure_app_save(&wide) == PURE_OK);
+    pure_app_stop(&wide);
+    CHECK(pure_app_start(&wide, &module_a, pure_tree_data(deep_path)) == PURE_OK && pure_list_count(wide.lists[1]) == 2 &&
+          pure_list_get(wide.lists[1], 1, 0).num == 7);
+    pure_app_stop(&wide);
+    remove_store(deep_path);
     static unsigned char bytes[65536];
     size_t size = build(bytes, (Spec){s1, 1, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0});
     CHECK(pure_module_load(bytes, size - 1, &module_a, error, sizeof(error)) == PURE_INVALID);