027's 4f-1: a module's tables sized to it: PureModule's thirty-four tables of states, inputs, message fields, views (the columns the load makes among them, counted from the view and record entries first), handlers, records, record fields, functions and signature items point into one block the load lays out (lay_tables) and pure_module_release gives back, so PureModule is 2,752 bytes, not 24,320; a sanitized build poisons 16 bytes between tables, so reading past one is caught (none was); limits as they were; a test and five of research 044's timers gave the load a module never zeroed (it gives back what one holds), now zeroed; counts: b38 against b37 with batches within 0.25%, without batches matmul -0.45%, join +0.36%, knapsack +0.28%, BFS -0.25%; the bank's queries (s37 and s38 on one aup module in turns, three each) field -0.68%, found -0.51%, large -0.47%, the rest -0.11% to -0.20%; 027's 4f designed in four steps and 4f-1 as made

3 files changed, 114 insertions, 41 deletions

diff --git a/core/module.c b/core/module.c
index e7118f0..f388ae7 100644
--- a/core/module.c
+++ b/core/module.c
@@ -1563,7 +1563,59 @@ PURE_COLD PureResult pure_module_read(const char *path, PureModule *out, char *e
     return result;
 }
 
+#ifdef __SANITIZE_ADDRESS__
+#include <sanitizer/asan_interface.h>
+enum { TABLE_GAP = 16 }; /* poisoned between tables: reading past one is caught */
+#else
+enum { TABLE_GAP = 0 };
+#endif
+
+/* The module's tables laid out from base (027's 4f-1), each a run of its entries aligned to 8; the bytes they take. */
+static size_t lay_tables(PureModule *m, uintptr_t base, size_t views, size_t signatures) {
+    size_t at = 0;
+#define LAY(member, count) \
+    (m->member = (void *)(base + at), at += (((size_t)(count) * sizeof(*m->member) + 7) & ~(size_t)7) + TABLE_GAP)
+    LAY(state_names, m->states), LAY(state_type, m->states), LAY(state_ref, m->states), LAY(state_flags, m->states);
+    LAY(state_key, m->states), LAY(state_indexed, m->states), LAY(initial, m->states);
+    LAY(input_names, m->inputs), LAY(input_keys, m->inputs), LAY(input, m->inputs), LAY(field, m->fields);
+    LAY(view_names, views), LAY(view_type, views), LAY(view_table, views), LAY(view_list, views), LAY(view_record, views);
+    LAY(view_field, views), LAY(view_computed, views), LAY(view_code, views), LAY(view_plan, views);
+    LAY(view_arguments, views), LAY(view_body, views), LAY(view_lists, views), LAY(view_range, views);
+    LAY(handler, m->handlers);
+    LAY(record_names, m->records), LAY(record, m->records), LAY(record_nested, m->records), LAY(record_held, m->records);
+    LAY(record_field_names, m->record_fields), LAY(record_field_type, m->record_fields);
+    LAY(record_field_default, m->record_fields);
+    LAY(function, m->functions), LAY(signature_type, signatures), LAY(signature_record, signatures);
+#undef LAY
+    return at;
+}
+
+static bool make_tables(PureModule *m, size_t views, size_t signatures) {
+    size_t size = lay_tables(m, 0, views, signatures);
+    unsigned char *base = calloc(1, size ? size : 1);
+    if (!base) return false;
+    m->tables = base;
+    lay_tables(m, (uintptr_t)base, views, signatures);
+#ifdef __SANITIZE_ADDRESS__
+    for (size_t at = 0; at + TABLE_GAP <= size;) { /* each run's gap, found again as it was laid out */
+        unsigned char *next = base + size;
+#define GAP(member) if ((unsigned char *)m->member > base + at && (unsigned char *)m->member < next) next = (unsigned char *)m->member;
+        GAP(state_type) GAP(state_ref) GAP(state_flags) GAP(state_key) GAP(state_indexed) GAP(initial) GAP(input_names)
+        GAP(input_keys) GAP(input) GAP(field) GAP(view_names) GAP(view_type) GAP(view_table) GAP(view_list) GAP(view_record)
+        GAP(view_field) GAP(view_computed) GAP(view_code) GAP(view_plan) GAP(view_arguments) GAP(view_body) GAP(view_lists)
+        GAP(view_range) GAP(handler) GAP(record_names) GAP(record) GAP(record_nested) GAP(record_held) GAP(record_field_names)
+        GAP(record_field_type) GAP(record_field_default) GAP(function) GAP(signature_type) GAP(signature_record)
+#undef GAP
+        ASAN_POISON_MEMORY_REGION(next - TABLE_GAP, TABLE_GAP);
+        at = (size_t)(next - base);
+    }
+#endif
+    return true;
+}
+
 void pure_module_release(PureModule *m) {
+    free(m->tables);
+    m->tables = NULL;
     pure_jit_free(m->jit);
     m->jit = NULL;
     m->batches = m->calls = false;
@@ -1664,6 +1716,23 @@ PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule
     out->constant_offset = (uint32_t *)(out->code + out->code_count);
     out->constant_length = out->constant_offset + out->constants;
     out->constant_data = (unsigned char *)(out->constant_length + out->constants);
+    /* Its tables, the columns the load makes among the views (027's 4e: a record's fields for each list view of a
+     * record), counted from the bytes first. */
+    size_t records_at = functions_at - (size_t)out->views * VIEW_BYTES - (size_t)out->handlers * HANDLER_BYTES -
+                        (size_t)out->record_fields * RECORD_FIELD_BYTES - (size_t)out->records * RECORD_BYTES;
+    size_t made = 0, views_at = records_at + (size_t)out->records * RECORD_BYTES + (size_t)out->record_fields * RECORD_FIELD_BYTES;
+    for (uint32_t i = 0; i < out->views; ++i) {
+        const unsigned char *v = p + views_at + (size_t)i * VIEW_BYTES;
+        uint32_t record = read16(v + 8);
+        if (v[4] == PURE_VIEW_LIST && !read16(v + 6) && record && record <= out->records)
+            made += read16(p + records_at + (size_t)(record - 1) * RECORD_BYTES + 6);
+    }
+    if (out->views + made > PURE_MODULE_VIEWS) REFUSE("views and their columns", out->views);
+    size_t capacity = out->views + made;
+    if (!make_tables(out, capacity, signatures)) {
+        result = PURE_ALLOCATION;
+        REFUSE("out of memory", 0);
+    }
     const unsigned char *at = p + PURE_MODULE_HEADER;
     const unsigned char *text = p + size - constant_bytes - strings;
     if (constant_bytes) memcpy(out->constant_data, p + size - constant_bytes, constant_bytes);
@@ -1879,7 +1948,7 @@ PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule
         if (!list_view(out, i) || !out->view_record[i]) continue;
         const PureRecord *r = &out->record[out->view_record[i] - 1];
         for (unsigned f = 0; f < r->count; ++f) {
-            if (out->views == PURE_MODULE_VIEWS) REFUSE("views and their columns", i);
+            if (out->views == capacity) REFUSE("views and their columns", i);
             uint32_t c = out->views++, field = r->first + f;
             memcpy(out->view_names[c], out->record_field_names[field], PURE_MODULE_NAME);
             out->view_type[c] = out->record_field_type[field];
@@ -2130,7 +2199,7 @@ refused:
 #undef REFUSE
 }
 
-static int find(const char (*names)[PURE_MODULE_NAME], unsigned count, const char *name) {
+static int find(char (*names)[PURE_MODULE_NAME], unsigned count, const char *name) {
     for (unsigned i = 0; i < count; ++i)
         if (!strcmp(names[i], name)) return (int)i;
     return -1;
diff --git a/core/module.h b/core/module.h
index 20e5fba..8ea0574 100644
--- a/core/module.h
+++ b/core/module.h
@@ -367,18 +367,22 @@ _Static_assert(PURE_MODULE_STATES <= 32, "a state is a bit of PureApp.counted");
 typedef struct {
     uint16_t states, inputs, views, handlers, fields, constants, records, record_fields;
     uint32_t code_count;
-    char state_names[PURE_MODULE_STATES][PURE_MODULE_NAME];
-    char input_names[PURE_MODULE_INPUTS][PURE_MODULE_NAME];
-    char view_names[PURE_MODULE_VIEWS][PURE_MODULE_NAME];
-    char input_keys[PURE_MODULE_INPUTS][PURE_MODULE_NAME];
-    uint8_t state_type[PURE_MODULE_STATES];
-    uint16_t state_ref[PURE_MODULE_STATES]; /* text: constant; cursor: text; list: record; index: list */
-    uint8_t state_flags[PURE_MODULE_STATES]; /* cursor: PURE_CURSOR_STAYS; list: PURE_LIST_ROWS >> 8 */
+    /* Its tables (027's 4f-1), each sized to it in one block (tables) that load takes and pure_module_release gives
+     * back: a state's, an input's, a message field's, a view's (the columns the load makes among them), a handler's, a
+     * record's, a record field's, a function's and a signature item's entries. */
+    void *tables;
+    char (*state_names)[PURE_MODULE_NAME];
+    char (*input_names)[PURE_MODULE_NAME];
+    char (*view_names)[PURE_MODULE_NAME];
+    char (*input_keys)[PURE_MODULE_NAME];
+    uint8_t *state_type;
+    uint16_t *state_ref; /* text: constant; cursor: text; list: record; index: list */
+    uint8_t *state_flags; /* cursor: PURE_CURSOR_STAYS; list: PURE_LIST_ROWS >> 8 */
     uint32_t transient; /* a bit per transient state */
     /* A keyed list: its key field + 1 (0: none). Its items stay in key order
      * (numbers by value, texts by bytes), each key once, and keys do not change. */
-    uint8_t state_key[PURE_MODULE_STATES];
-    bool state_indexed[PURE_MODULE_STATES]; /* list: an index state refers to it (set at load) */
+    uint8_t *state_key;
+    bool *state_indexed; /* list: an index state refers to it (set at load) */
     uint8_t *height;    /* the operand stack's height at each instruction, from its block's start (the verifier's) */
     /* In a batch's block (module-batch.c), a value computed again by the same code as an earlier one whose operands
      * nothing changed between: at its code's first instruction, the earlier root + 1 and its code's length (reuse_from,
@@ -391,45 +395,45 @@ typedef struct {
     bool batches;        /* a loop of it is planned to run in batches (module-batch.c, docs/design/024) */
     uint8_t runner;      /* which executor runs its blocks: 0 run's own steps, 1 with batches, 2 with calls (module.c) */
     bool held;           /* it holds bytes or texts by name (BYTES_STORE): events and views empty them as they begin */
-    PureInputShape input[PURE_MODULE_INPUTS];
-    PureField field[PURE_MODULE_FIELDS];
-    PureValue initial[PURE_MODULE_STATES];
-    uint8_t view_type[PURE_MODULE_VIEWS];
-    uint16_t view_table[PURE_MODULE_VIEWS]; /* a column's list view + 1, or 0 */
-    uint16_t view_list[PURE_MODULE_VIEWS];  /* the list state of a list view over one, or its column's */
-    uint16_t view_record[PURE_MODULE_VIEWS]; /* a list view's or a list column's items' record + 1 (a map's: 0) */
-    uint8_t view_field[PURE_MODULE_VIEWS];   /* a column the load made: its item's field + 1 (read as it is), or 0 */
-    bool view_computed[PURE_MODULE_VIEWS];   /* a list view whose code leaves its list (a column's: its view's) */
-    PureBlock view_code[PURE_MODULE_VIEWS];
-    uint8_t view_plan[PURE_MODULE_VIEWS]; /* a table's filter runs a leaf of its list at once (as a fold may) */
-    uint8_t view_arguments[PURE_MODULE_VIEWS]; /* the input of a view's arguments + 1 (a column's: its table's), or 0 */
+    PureInputShape *input;
+    PureField *field;
+    PureValue *initial;
+    uint8_t *view_type;
+    uint16_t *view_table; /* a column's list view + 1, or 0 */
+    uint16_t *view_list;  /* the list state of a list view over one, or its column's */
+    uint16_t *view_record; /* a list view's or a list column's items' record + 1 (a map's: 0) */
+    uint8_t *view_field;   /* a column the load made: its item's field + 1 (read as it is), or 0 */
+    bool *view_computed;   /* a list view whose code leaves its list (a column's: its view's) */
+    PureBlock *view_code;
+    uint8_t *view_plan; /* a table's filter runs a leaf of its list at once (as a fold may) */
+    uint8_t *view_arguments; /* the input of a view's arguments + 1 (a column's: its table's), or 0 */
     /* Set at load: a view's code computes (its locals and loops: a body), and the event's lists it changes, a bit each. */
-    bool view_body[PURE_MODULE_VIEWS];
-    uint32_t view_lists[PURE_MODULE_VIEWS];
+    bool *view_body;
+    uint32_t *view_lists;
     PureShape shape[PURE_MODULE_SHAPES];
     uint8_t shapes;
     PureRange range[PURE_MODULE_RANGES];
     uint8_t ranges;
-    uint8_t view_range[PURE_MODULE_VIEWS]; /* a table's filter's PureRange + 1, or 0 */
-    PureHandler handler[PURE_MODULE_HANDLERS];
+    uint8_t *view_range; /* a table's filter's PureRange + 1, or 0 */
+    PureHandler *handler;
     /* The code and the text constants, sized to the module in one block
      * (storage) that load takes and pure_module_release gives back. */
     PureOp *code;
     uint32_t *constant_offset, *constant_length;
     unsigned char *constant_data;
     void *storage;
-    char record_names[PURE_MODULE_RECORDS][PURE_MODULE_NAME];
-    PureRecord record[PURE_MODULE_RECORDS];
-    char record_field_names[PURE_MODULE_RECORD_FIELDS][PURE_MODULE_NAME];
-    uint8_t record_field_type[PURE_MODULE_RECORD_FIELDS]; /* PureRecordType */
-    PureValue record_field_default[PURE_MODULE_RECORD_FIELDS]; /* a number (a bool's 0 or 1), an i64, a text constant's index
-                                                                  * or a list's record (LIST, its i64) */
-    bool record_nested[PURE_MODULE_RECORDS]; /* a record with a LIST field (set at load) */
-    uint16_t record_held[PURE_MODULE_RECORDS]; /* its fields held as bytes are (pure_record_bytes), a bit each (set at load) */
+    char (*record_names)[PURE_MODULE_NAME];
+    PureRecord *record;
+    char (*record_field_names)[PURE_MODULE_NAME];
+    uint8_t *record_field_type; /* PureRecordType */
+    PureValue *record_field_default; /* a number (a bool's 0 or 1), an i64, a text constant's index or a list's record
+                                      * (LIST, its i64) */
+    bool *record_nested; /* a record with a LIST field (set at load) */
+    uint16_t *record_held; /* its fields held as bytes are (pure_record_bytes), a bit each (set at load) */
     uint16_t functions;
-    PureFunction function[PURE_MODULE_FUNCTIONS];
-    uint8_t signature_type[PURE_MODULE_SIGNATURES];
-    uint16_t signature_record[PURE_MODULE_SIGNATURES];
+    PureFunction *function;
+    uint8_t *signature_type;
+    uint16_t *signature_record;
     bool calls; /* it has functions: its blocks run in the executor that calls (module.c) */
 } PureModule;
 
diff --git a/core/test-module.c b/core/test-module.c
index 8950018..8014d63 100644
--- a/core/test-module.c
+++ b/core/test-module.c
@@ -3483,7 +3483,7 @@ static void lane_folds(void) {
             module_b.code[i].plan = module_b.code[i].shape = 0;
         }
         planned += module_a.view_plan[VIEWS];
-        memset(module_b.view_plan, 0, sizeof(module_b.view_plan));
+        memset(module_b.view_plan, 0, module_b.views * sizeof(*module_b.view_plan));
         pure_jit_free(module_b.jit); /* its code changed: compiled again (021) */
         module_b.jit = pure_jit_make(&module_b);
         PureApp a, b;
@@ -4079,7 +4079,7 @@ static void ranges(void) {
           !module_a.code[45].range && module_a.view_range[6] && module_a.code[78].range && !module_a.code[102].range &&
           !module_a.code[118].range && module_a.code[118].plan);
     for (uint32_t i = 0; i < module_b.code_count; ++i) module_b.code[i].range = 0;
-    memset(module_b.view_range, 0, sizeof(module_b.view_range));
+    memset(module_b.view_range, 0, module_b.views * sizeof(*module_b.view_range));
     pure_jit_free(module_b.jit); /* its code changed: compiled again (021) */
     module_b.jit = pure_jit_make(&module_b);
     uint32_t seed = 0x7a45e11u;
@@ -5426,7 +5426,7 @@ static void output_meaning(void) {
                 NULL, 0};
     Spec none = {nums, 1, none_inputs, 1, NULL, 0, none_handlers, 1, none_code, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0,
                  NULL, 0, NULL, 0};
-    PureModule without;
+    PureModule without = {0};
     CHECK(load(was, &module_a) == PURE_OK && load(now, &module_b) == PURE_OK && load(none, &without) == PURE_OK);
     CHECK(pure_module_shape(&module_a, 1) != pure_module_shape(&module_b, 1));
     char *path = new_store();