Size a module's code and constants to the module; raise their limits

A real app (a board, a stopwatch) already used a quarter of the 512
instructions, since a fn is inlined where it is called, and every text
it shows is one of 64 constants. The code and the constants now live in
one block sized to the module, which load takes and pure_module_release
gives back; the limits rise to 65,535 instructions and 4,096 constants
of up to 1 MiB, while PureModule shrinks from 28,664 to 11,936 bytes.
The verifier's branch marks are sized to the block instead of 64 KiB on
the stack. pure_module_read reads a module file whole, and every host
uses it instead of a 128 KiB buffer.

5 files changed, 89 insertions, 43 deletions

diff --git a/prototypes/connections/edit-module.c b/prototypes/connections/edit-module.c
index 9305923..3ce272e 100644
--- a/prototypes/connections/edit-module.c
+++ b/prototypes/connections/edit-module.c
@@ -89,19 +89,11 @@ int main(int argc, char **argv) {
     if (!show && !number(argv[at + 2], INT64_MAX, &expected)) goto usage;
     if (edit && (!number(argv[at + 3], UINT32_MAX, &offset) || !number(argv[at + 4], UINT32_MAX, &remove))) goto usage;
 
-    static unsigned char image[1 << 17];
     static PureModule module;
     char error[160];
-    FILE *source = fopen(module_path, "rb");
-    size_t image_size = source ? fread(image, 1, sizeof(image), source) : 0;
-    if (!source || ferror(source) || image_size == sizeof(image)) {
-        if (source) fclose(source);
-        fputs("Module: cannot read module\n", stderr);
-        return 2;
-    }
-    fclose(source);
-    if (pure_module_load(image, image_size, &module, error, sizeof(error)) != PURE_OK) {
-        fprintf(stderr, "Module: %s\n", error);
+    PureResult read = pure_module_read(module_path, &module, error, sizeof(error));
+    if (read != PURE_OK) {
+        fprintf(stderr, "Module: %s\n", read == PURE_NOT_FOUND ? "cannot read module" : error);
         return 2;
     }
     int input = pure_module_input(&module, show ? "show" : "edit");
diff --git a/prototypes/connections/module.c b/prototypes/connections/module.c
index 73a9ed6..1b96ac1 100644
--- a/prototypes/connections/module.c
+++ b/prototypes/connections/module.c
@@ -206,17 +206,19 @@ static bool same_stack(const Join *join, int height, const unsigned char *types,
  * in it); the block around treats it as one instruction and may not branch
  * into it. Returns why the block is refused, with at moved to the
  * instruction when one is at fault, or NULL. */
-PURE_COLD static const char *verify_block(const PureModule *m, PureBlock block, int view, int row, uint32_t event,
-                                uint32_t *at) {
+PURE_COLD static const char *verify_block(const PureModule *, PureBlock, int view, int row, uint32_t event, uint32_t *at);
+
+/* target marks, per instruction of the block, a branch into it with an empty stack. */
+PURE_COLD static const char *verify_code(const PureModule *m, PureBlock block, int view, int row, uint32_t event,
+                                         uint32_t *at, unsigned char *target) {
 #define REFUSE(why) do { *at = pc; return why; } while (0)
-    if (block.start > m->code_count || block.length > m->code_count - block.start) return "code range outside module";
     if (view >= 0 && !block.length) return "empty view";
     int input = view >= 0 ? -1 : event_input(event);
     const PureInputShape *shape = input >= 0 ? &m->input[input] : NULL;
     unsigned where = view >= 0 ? IN_VIEW | (row >= 0 ? IN_ROW : 0) : IN_HANDLER;
     if (shape && shape->kind == PURE_KIND_MESSAGE) where |= IN_MESSAGE | (shape->keyed ? IN_KEYED : 0);
     if (shape && shape->kind == PURE_KIND_TEXT) where |= IN_TEXT;
-    unsigned char types[PURE_MODULE_STACK], owners[PURE_MODULE_STACK], target[PURE_MODULE_CODE + 1] = {0};
+    unsigned char types[PURE_MODULE_STACK], owners[PURE_MODULE_STACK];
     Join joins[PURE_MODULE_JOINS];
     unsigned open = 0;
     int height = 0;
@@ -360,6 +362,49 @@ PURE_COLD static const char *verify_block(const PureModule *m, PureBlock block,
 #undef REFUSE
 }
 
+PURE_COLD static const char *verify_block(const PureModule *m, PureBlock block, int view, int row, uint32_t event,
+                                          uint32_t *at) {
+    if (block.start > m->code_count || block.length > m->code_count - block.start) return "code range outside module";
+    unsigned char *target = calloc((size_t)block.length + 1, 1);
+    if (!target) return "out of memory";
+    const char *why = verify_code(m, block, view, row, event, at, target);
+    free(target);
+    return why;
+}
+
+PURE_COLD PureResult pure_module_read(const char *path, PureModule *out, char *error, size_t error_size) {
+    FILE *file = fopen(path, "rb");
+    unsigned char *bytes = NULL;
+    size_t size = 0, capacity = 0;
+    bool ok = file != NULL;
+    /* The largest module is some 2 MiB: code, strings and constants at their limits. */
+    while (ok && !feof(file) && capacity < (8u << 20)) {
+        capacity = capacity ? capacity * 2 : 1 << 16;
+        unsigned char *grown = realloc(bytes, capacity);
+        if (!grown) { ok = false; break; }
+        bytes = grown;
+        size += fread(bytes + size, 1, capacity - size, file);
+        ok = !ferror(file);
+    }
+    ok = ok && feof(file);
+    if (file) fclose(file);
+    PureResult result = PURE_NOT_FOUND;
+    if (!ok) {
+        if (error_size) snprintf(error, error_size, "cannot read %s", path);
+        pure_module_release(out);
+    } else result = pure_module_load(bytes, size, out, error, error_size);
+    free(bytes);
+    return result;
+}
+
+void pure_module_release(PureModule *m) {
+    free(m->storage);
+    m->storage = NULL;
+    m->code = NULL;
+    m->constant_offset = m->constant_length = NULL;
+    m->constant_data = NULL;
+}
+
 PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule *out,
                             char *error, size_t error_size) {
     const unsigned char *p = bytes;
@@ -368,6 +413,7 @@ PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule
     PureResult result = PURE_INVALID;
 #define REFUSE(message, index) do { why = message; bad = (uint32_t)(index); goto refused; } while (0)
     if (error_size) error[0] = 0;
+    pure_module_release(out);
     memset(out, 0, sizeof(*out));
     if (!p || size < PURE_MODULE_HEADER || memcmp(p, magic, sizeof(magic))) REFUSE("not a Pure module v5", 0);
     out->states = (uint16_t)read16(p + 8);
@@ -399,9 +445,20 @@ PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule
         (size_t)out->views * VIEW_BYTES + (size_t)out->handlers * HANDLER_BYTES +
         (size_t)out->code_count * OP_BYTES + strings + constant_bytes;
     if (size != expected) REFUSE("module size", expected);
+    /* The code, the constants' offsets and lengths, then their bytes, in one block. */
+    size_t storage = (size_t)out->code_count * sizeof(PureOp) + (size_t)out->constants * 2 * sizeof(uint32_t) +
+                     constant_bytes;
+    if (!(out->storage = malloc(storage ? storage : 1))) {
+        result = PURE_ALLOCATION;
+        REFUSE("out of memory", 0);
+    }
+    out->code = out->storage;
+    out->constant_offset = (uint32_t *)(out->code + out->code_count);
+    out->constant_length = out->constant_offset + out->constants;
+    out->constant_data = (unsigned char *)(out->constant_length + out->constants);
     const unsigned char *at = p + PURE_MODULE_HEADER;
     const unsigned char *text = p + size - constant_bytes - strings;
-    memcpy(out->constant_data, p + size - constant_bytes, constant_bytes);
+    if (constant_bytes) memcpy(out->constant_data, p + size - constant_bytes, constant_bytes);
     for (uint32_t i = 0; i < out->states; ++i, at += STATE_BYTES) {
         out->state_type[i] = (uint8_t)read32(at + 4);
         if (!name_at(text, strings, read32(at), true, out->state_names[i])) REFUSE("state name", i);
@@ -579,6 +636,7 @@ PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule
     return PURE_OK;
 refused:
     if (error_size) snprintf(error, error_size, "%s (%u)", why, bad);
+    pure_module_release(out);
     return result;
 #undef REFUSE
 }
diff --git a/prototypes/connections/module.h b/prototypes/connections/module.h
index 95483c7..c3edc54 100644
--- a/prototypes/connections/module.h
+++ b/prototypes/connections/module.h
@@ -15,10 +15,10 @@
  * branches, names, records, tables, constants and effects. */
 enum {
     PURE_MODULE_STATES = 32, PURE_MODULE_INPUTS = 32, PURE_MODULE_VIEWS = 64,
-    PURE_MODULE_HANDLERS = 64, PURE_MODULE_CODE = 512, PURE_MODULE_STACK = 64,
+    PURE_MODULE_HANDLERS = 64, PURE_MODULE_CODE = 65535, PURE_MODULE_STACK = 64,
     PURE_MODULE_NAME = 32, PURE_MODULE_STRINGS = 32768, PURE_MODULE_HEADER = 40,
-    PURE_MODULE_FIELDS = 32, PURE_MODULE_COLLECTIONS = 1, PURE_MODULE_CONSTANTS = 64,
-    PURE_MODULE_CONSTANT_BYTES = 8192, PURE_MODULE_RECORDS = 8, PURE_MODULE_RECORD_FIELDS = 64,
+    PURE_MODULE_FIELDS = 32, PURE_MODULE_COLLECTIONS = 1, PURE_MODULE_CONSTANTS = 4096,
+    PURE_MODULE_CONSTANT_BYTES = 1 << 20, PURE_MODULE_RECORDS = 8, PURE_MODULE_RECORD_FIELDS = 64,
     PURE_RECORD_FIELDS = 16, PURE_MODULE_JOINS = 8, PURE_APP_PENDING = 256
 };
 
@@ -118,19 +118,28 @@ typedef struct {
     uint16_t view_list[PURE_MODULE_VIEWS];  /* the list of a table or column */
     PureBlock view_code[PURE_MODULE_VIEWS];
     PureHandler handler[PURE_MODULE_HANDLERS];
-    PureOp code[PURE_MODULE_CODE];
-    uint32_t constant_offset[PURE_MODULE_CONSTANTS], constant_length[PURE_MODULE_CONSTANTS];
+    /* The code and the text constants, sized to the module in one block
+     * (storage) that load takes and pure_module_release gives back. */
+    PureOp *code;
+    uint32_t *constant_offset, *constant_length;
+    unsigned char *constant_data;
+    void *storage;
     char record_names[PURE_MODULE_RECORDS][PURE_MODULE_NAME];
     PureRecord record[PURE_MODULE_RECORDS];
     char record_field_names[PURE_MODULE_RECORD_FIELDS][PURE_MODULE_NAME];
     uint8_t record_field_text[PURE_MODULE_RECORD_FIELDS];
     double record_field_default[PURE_MODULE_RECORD_FIELDS]; /* number, or text constant index */
-    unsigned char constant_data[PURE_MODULE_CONSTANT_BYTES];
 } PureModule;
 
-/* Decode and verify. On failure out is unspecified and error names the rule. */
+/* Decode and verify into out, which must be zeroed (a static one is) or hold
+ * a loaded module: what it held is given back first. On failure out holds
+ * nothing and error names the rule. */
 PureResult pure_module_load(const void *bytes, size_t size, PureModule *out,
                             char *error, size_t error_size);
+/* Give back a loaded module's code and constants (no app may still run it). */
+void pure_module_release(PureModule *);
+/* Read a module file whole and load it (NOT_FOUND when it cannot be read). */
+PureResult pure_module_read(const char *path, PureModule *out, char *error, size_t error_size);
 /* Index of a name, or -1. Views are those outside tables. */
 int pure_module_state(const PureModule *, const char *name);
 int pure_module_input(const PureModule *, const char *name);
diff --git a/prototypes/connections/pure-run.c b/prototypes/connections/pure-run.c
index ef36965..cfc3b5c 100644
--- a/prototypes/connections/pure-run.c
+++ b/prototypes/connections/pure-run.c
@@ -18,18 +18,10 @@ static const char *const fault_names[] = {"none", "divide", "non_finite", "clamp
 /* time MS sets the time events see (milliseconds since 1970); tick NAME is a timer's tick. */
 
 static bool load(const char *path, PureModule *module) {
-    FILE *file = fopen(path, "rb");
-    static unsigned char bytes[1 << 17];
-    size_t size = file ? fread(bytes, 1, sizeof(bytes), file) : 0;
-    bool ok = file && !ferror(file) && size < sizeof(bytes);
-    if (file) fclose(file);
-    char error[128];
-    if (!ok) { fprintf(stderr, "cannot read %s\n", path); return false; }
-    if (pure_module_load(bytes, size, module, error, sizeof(error)) != PURE_OK) {
-        fprintf(stderr, "%s: %s\n", path, error);
-        return false;
-    }
-    return true;
+    char error[160];
+    if (pure_module_read(path, module, error, sizeof(error)) == PURE_OK) return true;
+    fprintf(stderr, "%s: %s\n", path, error);
+    return false;
 }
 
 static bool json_span(void *context, PureSpan span) {
diff --git a/prototypes/connections/serve.c b/prototypes/connections/serve.c
index a60bf44..469c487 100644
--- a/prototypes/connections/serve.c
+++ b/prototypes/connections/serve.c
@@ -1268,14 +1268,9 @@ static bool replies_borrow(const PureModule *m, uint32_t input) {
 /* ---- Main ------------------------------------------------------------- */
 
 static bool load_module(PureModule *m, const char *path, bool *documents, uint32_t *show_out, uint32_t *edit_out) {
-    static unsigned char bytes[1 << 17];
-    char error[128];
-    FILE *file = fopen(path, "rb");
-    size_t size = file ? fread(bytes, 1, sizeof(bytes), file) : 0;
-    bool read_ok = file && !ferror(file) && size < sizeof(bytes);
-    if (file) fclose(file);
-    if (!read_ok || pure_module_load(bytes, size, m, error, sizeof(error)) != PURE_OK) {
-        fprintf(stderr, "Module: %s\n", read_ok ? error : "cannot read");
+    char error[160];
+    if (pure_module_read(path, m, error, sizeof(error)) != PURE_OK) {
+        fprintf(stderr, "Module: %s\n", error);
         return false;
     }
     /* Documents are served when the module has show and edit, as keyed messages. */