Size a module's code and constants to the module; raise their limits

A real app (a board, a stopwatch) already used a quarter of the 512
instructions, since a fn is inlined where it is called, and every text
it shows is one of 64 constants. The code and the constants now live in
one block sized to the module, which load takes and pure_module_release
gives back; the limits rise to 65,535 instructions and 4,096 constants
of up to 1 MiB, while PureModule shrinks from 28,664 to 11,936 bytes.
The verifier's branch marks are sized to the block instead of 64 KiB on
the stack. pure_module_read reads a module file whole, and every host
uses it instead of a 128 KiB buffer.

6 files changed, 93 insertions, 45 deletions

diff --git a/docs/design/008-module-format.md b/docs/design/008-module-format.md
index 1039358..db597e7 100644
--- a/docs/design/008-module-format.md
+++ b/docs/design/008-module-format.md
@@ -111,8 +111,10 @@ v5에는 뒤에 집계(FOLD), 걸러진 표, 마크, 그리고 **키 있는 리�
 
 ## 로드할 때 검증하는 것
 
-- 머리말, 예약 바이트, 명령 패딩이 0인지. 한도(상태 32, 입력 32, 보기 64(열 포함), 핸들러 64, 명령 512,
-  필드 32, 컬렉션 1, 문자열 32 KiB)와 정확한 파일 크기.
+- 머리말, 예약 바이트, 명령 패딩이 0인지. 한도(상태 32, 입력 32, 보기 64(열 포함), 핸들러 64, 명령 65,535,
+  text 상수 4,096개·1 MiB, 필드 32, 컬렉션 1, 이름 문자열 32 KiB)와 정확한 파일 크기. 명령과 상수는 모듈 크기에 맞춘
+  블록 하나에 담긴다(`PureModule`의 다른 표는 고정 크기로 12 KiB). 그래서 한도를 올려도 작은 앱의 메모리는 그대로다.
+  호스트는 모듈 파일을 크기대로 읽는다(`pure_module_read`).
 - 이름은 ASCII 식별자 1–31바이트이고, 상태·레코드·컬렉션·입력·보기가 한 이름공간을 쓴다. 필드 이름은 메시지나
   레코드 안에서 유일하다. 레코드(최대 8)의 필드 범위는 차례로 이어지고 모두 합쳐 64개 이하다.
   키 입력의 키는 정해진 목록(`Space Enter` 화살표 `Backspace Delete Home End Tab Escape PageUp PageDown KeyA–Z Digit0–9`)
diff --git a/prototypes/connections/edit-module.c b/prototypes/connections/edit-module.c
index 9305923..3ce272e 100644
--- a/prototypes/connections/edit-module.c
+++ b/prototypes/connections/edit-module.c
@@ -89,19 +89,11 @@ int main(int argc, char **argv) {
     if (!show && !number(argv[at + 2], INT64_MAX, &expected)) goto usage;
     if (edit && (!number(argv[at + 3], UINT32_MAX, &offset) || !number(argv[at + 4], UINT32_MAX, &remove))) goto usage;
 
-    static unsigned char image[1 << 17];
     static PureModule module;
     char error[160];
-    FILE *source = fopen(module_path, "rb");
-    size_t image_size = source ? fread(image, 1, sizeof(image), source) : 0;
-    if (!source || ferror(source) || image_size == sizeof(image)) {
-        if (source) fclose(source);
-        fputs("Module: cannot read module\n", stderr);
-        return 2;
-    }
-    fclose(source);
-    if (pure_module_load(image, image_size, &module, error, sizeof(error)) != PURE_OK) {
-        fprintf(stderr, "Module: %s\n", error);
+    PureResult read = pure_module_read(module_path, &module, error, sizeof(error));
+    if (read != PURE_OK) {
+        fprintf(stderr, "Module: %s\n", read == PURE_NOT_FOUND ? "cannot read module" : error);
         return 2;
     }
     int input = pure_module_input(&module, show ? "show" : "edit");
diff --git a/prototypes/connections/module.c b/prototypes/connections/module.c
index 73a9ed6..1b96ac1 100644
--- a/prototypes/connections/module.c
+++ b/prototypes/connections/module.c
@@ -206,17 +206,19 @@ static bool same_stack(const Join *join, int height, const unsigned char *types,
  * in it); the block around treats it as one instruction and may not branch
  * into it. Returns why the block is refused, with at moved to the
  * instruction when one is at fault, or NULL. */
-PURE_COLD static const char *verify_block(const PureModule *m, PureBlock block, int view, int row, uint32_t event,
-                                uint32_t *at) {
+PURE_COLD static const char *verify_block(const PureModule *, PureBlock, int view, int row, uint32_t event, uint32_t *at);
+
+/* target marks, per instruction of the block, a branch into it with an empty stack. */
+PURE_COLD static const char *verify_code(const PureModule *m, PureBlock block, int view, int row, uint32_t event,
+                                         uint32_t *at, unsigned char *target) {
 #define REFUSE(why) do { *at = pc; return why; } while (0)
-    if (block.start > m->code_count || block.length > m->code_count - block.start) return "code range outside module";
     if (view >= 0 && !block.length) return "empty view";
     int input = view >= 0 ? -1 : event_input(event);
     const PureInputShape *shape = input >= 0 ? &m->input[input] : NULL;
     unsigned where = view >= 0 ? IN_VIEW | (row >= 0 ? IN_ROW : 0) : IN_HANDLER;
     if (shape && shape->kind == PURE_KIND_MESSAGE) where |= IN_MESSAGE | (shape->keyed ? IN_KEYED : 0);
     if (shape && shape->kind == PURE_KIND_TEXT) where |= IN_TEXT;
-    unsigned char types[PURE_MODULE_STACK], owners[PURE_MODULE_STACK], target[PURE_MODULE_CODE + 1] = {0};
+    unsigned char types[PURE_MODULE_STACK], owners[PURE_MODULE_STACK];
     Join joins[PURE_MODULE_JOINS];
     unsigned open = 0;
     int height = 0;
@@ -360,6 +362,49 @@ PURE_COLD static const char *verify_block(const PureModule *m, PureBlock block,
 #undef REFUSE
 }
 
+PURE_COLD static const char *verify_block(const PureModule *m, PureBlock block, int view, int row, uint32_t event,
+                                          uint32_t *at) {
+    if (block.start > m->code_count || block.length > m->code_count - block.start) return "code range outside module";
+    unsigned char *target = calloc((size_t)block.length + 1, 1);
+    if (!target) return "out of memory";
+    const char *why = verify_code(m, block, view, row, event, at, target);
+    free(target);
+    return why;
+}
+
+PURE_COLD PureResult pure_module_read(const char *path, PureModule *out, char *error, size_t error_size) {
+    FILE *file = fopen(path, "rb");
+    unsigned char *bytes = NULL;
+    size_t size = 0, capacity = 0;
+    bool ok = file != NULL;
+    /* The largest module is some 2 MiB: code, strings and constants at their limits. */
+    while (ok && !feof(file) && capacity < (8u << 20)) {
+        capacity = capacity ? capacity * 2 : 1 << 16;
+        unsigned char *grown = realloc(bytes, capacity);
+        if (!grown) { ok = false; break; }
+        bytes = grown;
+        size += fread(bytes + size, 1, capacity - size, file);
+        ok = !ferror(file);
+    }
+    ok = ok && feof(file);
+    if (file) fclose(file);
+    PureResult result = PURE_NOT_FOUND;
+    if (!ok) {
+        if (error_size) snprintf(error, error_size, "cannot read %s", path);
+        pure_module_release(out);
+    } else result = pure_module_load(bytes, size, out, error, error_size);
+    free(bytes);
+    return result;
+}
+
+void pure_module_release(PureModule *m) {
+    free(m->storage);
+    m->storage = NULL;
+    m->code = NULL;
+    m->constant_offset = m->constant_length = NULL;
+    m->constant_data = NULL;
+}
+
 PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule *out,
                             char *error, size_t error_size) {
     const unsigned char *p = bytes;
@@ -368,6 +413,7 @@ PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule
     PureResult result = PURE_INVALID;
 #define REFUSE(message, index) do { why = message; bad = (uint32_t)(index); goto refused; } while (0)
     if (error_size) error[0] = 0;
+    pure_module_release(out);
     memset(out, 0, sizeof(*out));
     if (!p || size < PURE_MODULE_HEADER || memcmp(p, magic, sizeof(magic))) REFUSE("not a Pure module v5", 0);
     out->states = (uint16_t)read16(p + 8);
@@ -399,9 +445,20 @@ PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule
         (size_t)out->views * VIEW_BYTES + (size_t)out->handlers * HANDLER_BYTES +
         (size_t)out->code_count * OP_BYTES + strings + constant_bytes;
     if (size != expected) REFUSE("module size", expected);
+    /* The code, the constants' offsets and lengths, then their bytes, in one block. */
+    size_t storage = (size_t)out->code_count * sizeof(PureOp) + (size_t)out->constants * 2 * sizeof(uint32_t) +
+                     constant_bytes;
+    if (!(out->storage = malloc(storage ? storage : 1))) {
+        result = PURE_ALLOCATION;
+        REFUSE("out of memory", 0);
+    }
+    out->code = out->storage;
+    out->constant_offset = (uint32_t *)(out->code + out->code_count);
+    out->constant_length = out->constant_offset + out->constants;
+    out->constant_data = (unsigned char *)(out->constant_length + out->constants);
     const unsigned char *at = p + PURE_MODULE_HEADER;
     const unsigned char *text = p + size - constant_bytes - strings;
-    memcpy(out->constant_data, p + size - constant_bytes, constant_bytes);
+    if (constant_bytes) memcpy(out->constant_data, p + size - constant_bytes, constant_bytes);
     for (uint32_t i = 0; i < out->states; ++i, at += STATE_BYTES) {
         out->state_type[i] = (uint8_t)read32(at + 4);
         if (!name_at(text, strings, read32(at), true, out->state_names[i])) REFUSE("state name", i);
@@ -579,6 +636,7 @@ PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule
     return PURE_OK;
 refused:
     if (error_size) snprintf(error, error_size, "%s (%u)", why, bad);
+    pure_module_release(out);
     return result;
 #undef REFUSE
 }
diff --git a/prototypes/connections/module.h b/prototypes/connections/module.h
index 95483c7..c3edc54 100644
--- a/prototypes/connections/module.h
+++ b/prototypes/connections/module.h
@@ -15,10 +15,10 @@
  * branches, names, records, tables, constants and effects. */
 enum {
     PURE_MODULE_STATES = 32, PURE_MODULE_INPUTS = 32, PURE_MODULE_VIEWS = 64,
-    PURE_MODULE_HANDLERS = 64, PURE_MODULE_CODE = 512, PURE_MODULE_STACK = 64,
+    PURE_MODULE_HANDLERS = 64, PURE_MODULE_CODE = 65535, PURE_MODULE_STACK = 64,
     PURE_MODULE_NAME = 32, PURE_MODULE_STRINGS = 32768, PURE_MODULE_HEADER = 40,
-    PURE_MODULE_FIELDS = 32, PURE_MODULE_COLLECTIONS = 1, PURE_MODULE_CONSTANTS = 64,
-    PURE_MODULE_CONSTANT_BYTES = 8192, PURE_MODULE_RECORDS = 8, PURE_MODULE_RECORD_FIELDS = 64,
+    PURE_MODULE_FIELDS = 32, PURE_MODULE_COLLECTIONS = 1, PURE_MODULE_CONSTANTS = 4096,
+    PURE_MODULE_CONSTANT_BYTES = 1 << 20, PURE_MODULE_RECORDS = 8, PURE_MODULE_RECORD_FIELDS = 64,
     PURE_RECORD_FIELDS = 16, PURE_MODULE_JOINS = 8, PURE_APP_PENDING = 256
 };
 
@@ -118,19 +118,28 @@ typedef struct {
     uint16_t view_list[PURE_MODULE_VIEWS];  /* the list of a table or column */
     PureBlock view_code[PURE_MODULE_VIEWS];
     PureHandler handler[PURE_MODULE_HANDLERS];
-    PureOp code[PURE_MODULE_CODE];
-    uint32_t constant_offset[PURE_MODULE_CONSTANTS], constant_length[PURE_MODULE_CONSTANTS];
+    /* The code and the text constants, sized to the module in one block
+     * (storage) that load takes and pure_module_release gives back. */
+    PureOp *code;
+    uint32_t *constant_offset, *constant_length;
+    unsigned char *constant_data;
+    void *storage;
     char record_names[PURE_MODULE_RECORDS][PURE_MODULE_NAME];
     PureRecord record[PURE_MODULE_RECORDS];
     char record_field_names[PURE_MODULE_RECORD_FIELDS][PURE_MODULE_NAME];
     uint8_t record_field_text[PURE_MODULE_RECORD_FIELDS];
     double record_field_default[PURE_MODULE_RECORD_FIELDS]; /* number, or text constant index */
-    unsigned char constant_data[PURE_MODULE_CONSTANT_BYTES];
 } PureModule;
 
-/* Decode and verify. On failure out is unspecified and error names the rule. */
+/* Decode and verify into out, which must be zeroed (a static one is) or hold
+ * a loaded module: what it held is given back first. On failure out holds
+ * nothing and error names the rule. */
 PureResult pure_module_load(const void *bytes, size_t size, PureModule *out,
                             char *error, size_t error_size);
+/* Give back a loaded module's code and constants (no app may still run it). */
+void pure_module_release(PureModule *);
+/* Read a module file whole and load it (NOT_FOUND when it cannot be read). */
+PureResult pure_module_read(const char *path, PureModule *out, char *error, size_t error_size);
 /* Index of a name, or -1. Views are those outside tables. */
 int pure_module_state(const PureModule *, const char *name);
 int pure_module_input(const PureModule *, const char *name);
diff --git a/prototypes/connections/pure-run.c b/prototypes/connections/pure-run.c
index ef36965..cfc3b5c 100644
--- a/prototypes/connections/pure-run.c
+++ b/prototypes/connections/pure-run.c
@@ -18,18 +18,10 @@ static const char *const fault_names[] = {"none", "divide", "non_finite", "clamp
 /* time MS sets the time events see (milliseconds since 1970); tick NAME is a timer's tick. */
 
 static bool load(const char *path, PureModule *module) {
-    FILE *file = fopen(path, "rb");
-    static unsigned char bytes[1 << 17];
-    size_t size = file ? fread(bytes, 1, sizeof(bytes), file) : 0;
-    bool ok = file && !ferror(file) && size < sizeof(bytes);
-    if (file) fclose(file);
-    char error[128];
-    if (!ok) { fprintf(stderr, "cannot read %s\n", path); return false; }
-    if (pure_module_load(bytes, size, module, error, sizeof(error)) != PURE_OK) {
-        fprintf(stderr, "%s: %s\n", path, error);
-        return false;
-    }
-    return true;
+    char error[160];
+    if (pure_module_read(path, module, error, sizeof(error)) == PURE_OK) return true;
+    fprintf(stderr, "%s: %s\n", path, error);
+    return false;
 }
 
 static bool json_span(void *context, PureSpan span) {
diff --git a/prototypes/connections/serve.c b/prototypes/connections/serve.c
index a60bf44..469c487 100644
--- a/prototypes/connections/serve.c
+++ b/prototypes/connections/serve.c
@@ -1268,14 +1268,9 @@ static bool replies_borrow(const PureModule *m, uint32_t input) {
 /* ---- Main ------------------------------------------------------------- */
 
 static bool load_module(PureModule *m, const char *path, bool *documents, uint32_t *show_out, uint32_t *edit_out) {
-    static unsigned char bytes[1 << 17];
-    char error[128];
-    FILE *file = fopen(path, "rb");
-    size_t size = file ? fread(bytes, 1, sizeof(bytes), file) : 0;
-    bool read_ok = file && !ferror(file) && size < sizeof(bytes);
-    if (file) fclose(file);
-    if (!read_ok || pure_module_load(bytes, size, m, error, sizeof(error)) != PURE_OK) {
-        fprintf(stderr, "Module: %s\n", read_ok ? error : "cannot read");
+    char error[160];
+    if (pure_module_read(path, m, error, sizeof(error)) != PURE_OK) {
+        fprintf(stderr, "Module: %s\n", error);
         return false;
     }
     /* Documents are served when the module has show and edit, as keyed messages. */