From 37d3a1e152707d2da587440c4cea8601b9ef0fd2 Mon Sep 17 00:00:00 2001 From: eunhhu Date: Sun, 27 Sep 2026 13:14:38 +0100 Subject: Lay Pure out as a core with the parts selected around it The app model, the tree store and the HTTP host lived in prototypes/connections among the earlier document runtime, the store was known only by its format's name, and Vega compiled six Pure sources by path. Now, as the README's principle has it: core/ runtime, flow and the app model (module*.c), with what the core asks of storage (state.h, store.h) store/ the tree store (PURETRE3, PUREREC1) serve/ pure-serve: HTTP host, outbound delivery, JSON in place cli/ pure-run, pure-edit-module A top-level make builds libpure.a (app model and store, for hosts to link) and the hosts; make check runs the store's and the app model's tests. The earlier runtime still builds from prototypes/connections against core/ and store/ (check, check-tree and check-core pass). Design 009 records the rule: parts by role behind libraries; git repositories split by coupling, so the store stays here while its contract moves with the core, and Vega stays apart. --- .gitignore | 1 + Makefile | 58 + README.md | 18 +- benchmarks/http/README.md | 2 +- benchmarks/http/run.py | 5 +- cli/edit-module.c | 179 ++ cli/pure-run.c | 196 ++ core/flow.c | 41 + core/flow.h | 25 + core/module-list.c | 428 ++++ core/module-list.h | 74 + core/module-save.c | 488 ++++ core/module-save.h | 36 + core/module-store.c | 88 + core/module-store.h | 11 + core/module-text.c | 189 ++ core/module-text.h | 34 + core/module.c | 2122 ++++++++++++++++ core/module.h | 343 +++ core/result.h | 8 + core/runtime.c | 394 +++ core/runtime.h | 150 ++ core/state.h | 84 + core/store.h | 54 + core/test-module.c | 2853 ++++++++++++++++++++++ docs/design/008-module-format.md | 4 +- docs/design/009-layout.md | 60 + docs/research/018-native-connections.md | 2 +- docs/research/019-byte-storage.md | 2 +- docs/research/030-authenticated-document-peer.md | 2 +- docs/research/064-release-callback-reentrancy.md | 2 +- docs/research/066-application-boundary.md | 2 +- docs/research/073-single-file-persistent-tree.md | 2 +- docs/research/074-runtime-owned-drafts.md | 4 +- docs/research/075-http-host-against-axum.md | 2 +- docs/research/076-one-sync-commit.md | 2 +- prototypes/connections/Makefile | 80 +- prototypes/connections/README.md | 5 + prototypes/connections/check.py | 8 +- prototypes/connections/deliver.c | 716 ------ prototypes/connections/deliver.h | 60 - prototypes/connections/edit-module.c | 179 -- prototypes/connections/flow.c | 41 - prototypes/connections/flow.h | 25 - prototypes/connections/message-json.c | 281 --- prototypes/connections/message-json.h | 26 - prototypes/connections/module-list.c | 428 ---- prototypes/connections/module-list.h | 74 - prototypes/connections/module-save.c | 488 ---- prototypes/connections/module-save.h | 36 - prototypes/connections/module-store.c | 88 - prototypes/connections/module-store.h | 11 - prototypes/connections/module-text.c | 189 -- prototypes/connections/module-text.h | 34 - prototypes/connections/module.c | 2122 ---------------- prototypes/connections/module.h | 343 --- prototypes/connections/package_source.py | 2 +- prototypes/connections/pure-run.c | 196 -- prototypes/connections/result.h | 8 - prototypes/connections/runtime.c | 394 --- prototypes/connections/runtime.h | 150 -- prototypes/connections/serve.c | 1820 -------------- prototypes/connections/state.h | 84 - prototypes/connections/store-tree.c | 1856 -------------- prototypes/connections/store.h | 54 - prototypes/connections/test-module.c | 2853 ---------------------- prototypes/connections/test-tree-store.c | 1131 --------- research/experiments/009-runtime-drafts/run.py | 8 +- research/experiments/010-one-sync-commit/run.py | 15 +- research/experiments/012-tail-records/run.py | 15 +- serve/deliver.c | 716 ++++++ serve/deliver.h | 60 + serve/message-json.c | 281 +++ serve/message-json.h | 26 + serve/serve.c | 1820 ++++++++++++++ store/store-tree.c | 1856 ++++++++++++++ store/test-tree-store.c | 1131 +++++++++ 77 files changed, 13909 insertions(+), 13766 deletions(-) create mode 100644 Makefile create mode 100644 cli/edit-module.c create mode 100644 cli/pure-run.c create mode 100644 core/flow.c create mode 100644 core/flow.h create mode 100644 core/module-list.c create mode 100644 core/module-list.h create mode 100644 core/module-save.c create mode 100644 core/module-save.h create mode 100644 core/module-store.c create mode 100644 core/module-store.h create mode 100644 core/module-text.c create mode 100644 core/module-text.h create mode 100644 core/module.c create mode 100644 core/module.h create mode 100644 core/result.h create mode 100644 core/runtime.c create mode 100644 core/runtime.h create mode 100644 core/state.h create mode 100644 core/store.h create mode 100644 core/test-module.c create mode 100644 docs/design/009-layout.md delete mode 100644 prototypes/connections/deliver.c delete mode 100644 prototypes/connections/deliver.h delete mode 100644 prototypes/connections/edit-module.c delete mode 100644 prototypes/connections/flow.c delete mode 100644 prototypes/connections/flow.h delete mode 100644 prototypes/connections/message-json.c delete mode 100644 prototypes/connections/message-json.h delete mode 100644 prototypes/connections/module-list.c delete mode 100644 prototypes/connections/module-list.h delete mode 100644 prototypes/connections/module-save.c delete mode 100644 prototypes/connections/module-save.h delete mode 100644 prototypes/connections/module-store.c delete mode 100644 prototypes/connections/module-store.h delete mode 100644 prototypes/connections/module-text.c delete mode 100644 prototypes/connections/module-text.h delete mode 100644 prototypes/connections/module.c delete mode 100644 prototypes/connections/module.h delete mode 100644 prototypes/connections/pure-run.c delete mode 100644 prototypes/connections/result.h delete mode 100644 prototypes/connections/runtime.c delete mode 100644 prototypes/connections/runtime.h delete mode 100644 prototypes/connections/serve.c delete mode 100644 prototypes/connections/state.h delete mode 100644 prototypes/connections/store-tree.c delete mode 100644 prototypes/connections/store.h delete mode 100644 prototypes/connections/test-module.c delete mode 100644 prototypes/connections/test-tree-store.c create mode 100644 serve/deliver.c create mode 100644 serve/deliver.h create mode 100644 serve/message-json.c create mode 100644 serve/message-json.h create mode 100644 serve/serve.c create mode 100644 store/store-tree.c create mode 100644 store/test-tree-store.c diff --git a/.gitignore b/.gitignore index c00450a..4d30289 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ __pycache__/ /benchmarks/http/axum-baseline/target/ /benchmarks/http/load /benchmarks/http/*.pmod +/build/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..50448da --- /dev/null +++ b/Makefile @@ -0,0 +1,58 @@ +CC ?= cc +AR ?= ar +CFLAGS ?= -O2 -g +CFLAGS += -std=c17 -Wall -Wextra -Wpedantic -Werror +BUILD ?= build + +# Pure is a small core with the rest selected around it: +# core/ execution, ownership, limits and completion (runtime), and the app +# model: modules verified before they run, events as transactions, +# states as drafts; state.h and store.h are what it asks of storage. +# store/ the tree store (one file; commits are one contiguous write and one +# sync; formats PURETRE3 and PUREREC1). +# serve/ pure-serve: the HTTP host, outbound delivery, JSON in place. +# cli/ command-line hosts. +# libpure.a is the app model with the tree store: what a host embeds (Vega's +# surfaces link it). prototypes/ keeps the earlier document runtime (000-077). +INCLUDES = -Icore +MODEL = core/module.c core/module-list.c core/module-save.c core/module-text.c core/module-store.c core/flow.c +STORE = store/store-tree.c +SERVE = serve/serve.c serve/deliver.c serve/message-json.c +HEADERS = $(wildcard core/*.h) $(wildcard serve/*.h) +OBJECTS = $(patsubst %.c,$(BUILD)/obj/%.o,$(MODEL) $(STORE)) + +.PHONY: all check +all: $(BUILD)/libpure.a $(BUILD)/pure-serve $(BUILD)/pure-run $(BUILD)/pure-edit-module + +$(BUILD)/obj/%.o: %.c $(HEADERS) + @mkdir -p $(dir $@) + $(CC) $(CPPFLAGS) $(CFLAGS) $(INCLUDES) -c $< -o $@ + +$(BUILD)/libpure.a: $(OBJECTS) + rm -f $@ + $(AR) rcs $@ $^ + +# The runtime's tickets and budgets, with effects completed off the owner thread. +$(BUILD)/pure-serve: $(SERVE) core/runtime.c $(BUILD)/libpure.a $(HEADERS) + $(CC) $(CPPFLAGS) $(CFLAGS) $(INCLUDES) -Iserve -DPURE_ASYNC_EFFECTS $(SERVE) core/runtime.c $(BUILD)/libpure.a \ + $(LDFLAGS) -lssl -lcrypto -lm -pthread $(LDLIBS) -o $@ + +$(BUILD)/pure-run: cli/pure-run.c $(BUILD)/libpure.a $(HEADERS) + $(CC) $(CPPFLAGS) $(CFLAGS) $(INCLUDES) cli/pure-run.c $(BUILD)/libpure.a $(LDFLAGS) -lm $(LDLIBS) -o $@ + +# The document CLI contract, driven by a module on the tree store. +$(BUILD)/pure-edit-module: cli/edit-module.c $(BUILD)/libpure.a $(HEADERS) + $(CC) $(CPPFLAGS) $(CFLAGS) $(INCLUDES) cli/edit-module.c $(BUILD)/libpure.a $(LDFLAGS) -lm $(LDLIBS) -o $@ + +# Tests build their own copies: the store's fault hooks exist only under PURE_STORE_TEST. +$(BUILD)/test-module: core/test-module.c $(MODEL) $(STORE) $(HEADERS) + @mkdir -p $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) $(INCLUDES) -DPURE_STORE_TEST core/test-module.c $(MODEL) $(STORE) $(LDFLAGS) -pthread -lm $(LDLIBS) -o $@ + +$(BUILD)/test-tree-store: store/test-tree-store.c $(STORE) $(HEADERS) + @mkdir -p $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) $(INCLUDES) -DPURE_STORE_TEST store/test-tree-store.c $(STORE) $(LDFLAGS) $(LDLIBS) -o $@ + +check: all $(BUILD)/test-module $(BUILD)/test-tree-store + $(BUILD)/test-tree-store + $(BUILD)/test-module diff --git a/README.md b/README.md index 3108736..c7c7958 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,22 @@ batches, and a receiving app applies each message once. Measurements against axu are in [075](docs/research/075-http-host-against-axum.md), [077](docs/research/077-apps-as-http-services.md), [080](docs/research/080-snapshot-reads.md), [081](docs/research/081-tail-records.md), [082](docs/research/082-json-in-place.md) and [083](docs/research/083-delivery-over-http.md); the status table is -[CHECKPOINT.md](CHECKPOINT.md). The text below -records the earlier document runtime this grew from. +[CHECKPOINT.md](CHECKPOINT.md). + +The source follows the principle below, a small core with the rest selected around it +([layout](docs/design/009-layout.md)): + +| Directory | What | Built as | +|---|---|---| +| `core/` | execution, ownership, limits and completion (`runtime.c`); the app model: modules verified before they run, events as transactions, states as drafts (`module*.c`); what the core asks of storage (`state.h`, `store.h`) | `build/libpure.a` (with the store) | +| `store/` | the tree store: one file, a commit is one contiguous write and one sync (formats `PURETRE3`, `PUREREC1`) | in `libpure.a` | +| `serve/` | `pure-serve`: the HTTP host, outbound delivery, JSON read in place | `build/pure-serve` | +| `cli/` | command-line hosts | `build/pure-run`, `build/pure-edit-module` | +| `prototypes/` | the earlier document runtime (000-077), built against `core/` and `store/` | `make -C prototypes/connections` | + +`make` builds the library and the hosts, `make check` runs the store's and the app model's tests. Interfaces for +people are [Vega](/srv/git/vega.git), which links `libpure.a`. The text below records the earlier document runtime +this grew from. A standalone native micro runtime being designed as a small operating system. Pure owns the application model for execution, memory, protection, diff --git a/benchmarks/http/README.md b/benchmarks/http/README.md index af9f62d..22fc6d3 100644 --- a/benchmarks/http/README.md +++ b/benchmarks/http/README.md @@ -7,7 +7,7 @@ Pure의 문서 서버들과 같은 계약의 axum(Rust)·drogon(C++) 서비스 cc -O2 -std=c17 -o load load.c (cd axum-baseline && cargo build --release) drogon-baseline/build.sh # Debian의 drogon 패키지를 설치하지 않고 .deps에 풀어 빌드한다 -make -C ../../prototypes/connections build/pure-serve +make -C ../.. build/pure-serve make -C ../../prototypes/connections BUILD=build/tree STORE=store-tree.c build/tree/pure-connections aup build ../../../aup/examples/editor.aup -o editor.pmod python3 run.py --module editor.pmod --directory /var/tmp # 저장소를 둘 장치에서 diff --git a/benchmarks/http/run.py b/benchmarks/http/run.py index bad9cc4..4605226 100644 --- a/benchmarks/http/run.py +++ b/benchmarks/http/run.py @@ -38,7 +38,8 @@ import http.client from pathlib import Path HERE = Path(__file__).resolve().parent -PURE = HERE.parent.parent / 'prototypes' / 'connections' +PURE = HERE.parent.parent # build/pure-serve; the earlier document server stays in prototypes/connections +PROTOTYPES = PURE / "prototypes" / "connections" def start(args): @@ -145,7 +146,7 @@ def main(): drogon = HERE / 'drogon-baseline/drogon-baseline' base = Path(tempfile.mkdtemp(dir=args.directory, prefix='pure-http-bench-')) servers = { - 'pure-program': lambda d: [PURE / 'build/tree/pure-connections', '--program', PURE / 'examples/editor.pure', + 'pure-program': lambda d: [PROTOTYPES / 'build/tree/pure-connections', '--program', PROTOTYPES / 'examples/editor.pure', '--store', d, '--port', '0', '--budget', str(args.budget)], 'pure-serve': lambda d: [PURE / 'build/pure-serve', '--module', args.module, '--store', d, '--port', '0', '--budget', str(args.budget)], diff --git a/cli/edit-module.c b/cli/edit-module.c new file mode 100644 index 0000000..3ce272e --- /dev/null +++ b/cli/edit-module.c @@ -0,0 +1,179 @@ +#define _POSIX_C_SOURCE 200809L +#include "module-store.h" +#include "store.h" +#include +#include +#include +#include + +/* The document CLI contract of edit.c, driven by a verified Pure module + * instead of a document program. The module declares keyed messages named + * show and edit; edit has u32 fields start and remove and a bytes tail. */ +static bool number(const char *text, uint64_t limit, uint64_t *out) { + if (!*text) return false; + uint64_t value = 0; + for (; *text; ++text) { + if (*text < '0' || *text > '9') return false; + unsigned digit = (unsigned)(*text - '0'); + if (digit > limit || value > (limit - digit) / 10) return false; + value = value * 10 + digit; + } + *out = value; + return true; +} + +static const char *message(PureResult status) { + switch (status) { + case PURE_OK: return "ok"; + case PURE_NOT_FOUND: return "not_found"; + case PURE_CONFLICT: return "conflict"; + case PURE_BUSY: return "busy"; + case PURE_BUDGET: return "payload_budget"; + case PURE_LIMIT: return "limit"; + case PURE_UNCERTAIN: return "durability_unknown"; + case PURE_DENIED: return "denied"; + case PURE_ALLOCATION: return "allocation"; + case PURE_INVALID: return "invalid"; + default: return "failed"; + } +} + +static bool valid_key(const char *key) { + size_t length = strlen(key); + if (!length || length > 64) return false; + for (size_t i = 0; i < length; ++i) { + char c = key[i]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '-' || c == '_')) return false; + } + return true; +} + +static void put32(unsigned char *to, uint32_t value) { + for (unsigned i = 0; i < 4; ++i) to[i] = (unsigned char)(value >> (8 * i)); +} + +static int field(const PureModule *m, int input, const char *name, uint8_t type) { + const PureInputShape *shape = &m->input[input]; + for (uint32_t i = 0; i < shape->field_count; ++i) + if (!strcmp(m->field[shape->field_start + i].name, name) && m->field[shape->field_start + i].type == type) + return (int)(shape->field_start + i); + return -1; +} + +static bool write_span(void *context, PureSpan span) { + return fwrite(span.data, 1, span.size, context) == span.size; +} + +static bool count_span(void *context, PureSpan span) { + *(size_t *)context += span.size; + return true; +} + +int main(int argc, char **argv) { + const char *module_path = NULL, *store_path = NULL; + size_t budget = 4u * 1024u * 1024u; + int at = 1; + for (; at + 1 < argc && !strncmp(argv[at], "--", 2); at += 2) { + uint64_t value; + if (!strcmp(argv[at], "--store")) store_path = argv[at + 1]; + else if (!strcmp(argv[at], "--module")) module_path = argv[at + 1]; + else if (!strcmp(argv[at], "--budget") && number(argv[at + 1], SIZE_MAX, &value)) budget = (size_t)value; + else goto usage; + } + if (!module_path || !store_path || at >= argc) goto usage; + bool show = !strcmp(argv[at], "show"), edit = !strcmp(argv[at], "edit"); + if ((!show && !edit && strcmp(argv[at], "apply")) || + argc - at != (show ? 2 : edit ? 6 : 4)) goto usage; + uint64_t expected = 0, offset = 0, remove = 0; + if (!show && !number(argv[at + 2], INT64_MAX, &expected)) goto usage; + if (edit && (!number(argv[at + 3], UINT32_MAX, &offset) || !number(argv[at + 4], UINT32_MAX, &remove))) goto usage; + + static PureModule module; + char error[160]; + PureResult read = pure_module_read(module_path, &module, error, sizeof(error)); + if (read != PURE_OK) { + fprintf(stderr, "Module: %s\n", read == PURE_NOT_FOUND ? "cannot read module" : error); + return 2; + } + int input = pure_module_input(&module, show ? "show" : "edit"); + if (input < 0 || module.input[input].kind != PURE_KIND_MESSAGE || !module.input[input].keyed) { + fprintf(stderr, "Module: needs a keyed message named %s\n", show ? "show" : "edit"); + return 2; + } + const PureInputShape *shape = &module.input[input]; + int start_field = field(&module, input, "start", PURE_FIELD_U32); + int remove_field = field(&module, input, "remove", PURE_FIELD_U32); + size_t prefix = edit ? shape->prefix : 0; + if (edit && (start_field < 0 || remove_field < 0 || !shape->field_count || + module.field[shape->field_start + shape->field_count - 1].type != PURE_FIELD_BYTES)) { + fputs("edit requires u32 fields start and remove and trailing insertion bytes; use apply for a custom input\n", stderr); + return 2; + } + FILE *file = NULL; + size_t length = 0; + if (!show) { + file = fopen(argv[at + (edit ? 5 : 3)], "rb"); + if (!file) { perror("input"); return 2; } + long bytes; + if (fseek(file, 0, SEEK_END) || (bytes = ftell(file)) < 0 || + (uintmax_t)bytes > PURE_OBJECT_LIMIT - prefix || fseek(file, 0, SEEK_SET)) { + fputs("input must be a seekable file within the request size limit\n", stderr); + fclose(file); return 2; + } + length = (size_t)bytes; + } + if (pure_store_init(store_path) != PURE_OK) { + if (file) fclose(file); + fputs("cannot initialize store\n", stderr); return 2; + } + const char *key = argv[at + 1]; + if (!valid_key(key) || prefix + length > budget) { + if (file) fclose(file); + fprintf(stderr, "%s\n", valid_key(key) ? "payload_budget" : "invalid"); return 2; + } + unsigned char *payload = NULL; + if (!show) { + payload = calloc(1, prefix + length ? prefix + length : 1); + if (!payload) { fclose(file); fputs("allocation\n", stderr); return 2; } + if (edit) { + put32(payload + module.field[start_field].offset, (uint32_t)offset); + put32(payload + module.field[remove_field].offset, (uint32_t)remove); + } + bool valid = !length || fread(payload + prefix, 1, length, file) == length; + if (fgetc(file) != EOF || ferror(file)) valid = false; + if (fclose(file)) valid = false; + if (!valid) { + free(payload); + fputs("input changed or could not be read; nothing was applied\n", stderr); return 2; + } + } + PureApp app; + if (pure_app_start(&app, &module, pure_tree_data(store_path)) != PURE_OK) { + free(payload); + fputs("allocation\n", stderr); return 2; + } + PureResult status = pure_app_message(&app, (uint32_t)input, key, show ? -1 : (int64_t)expected, + (PureSpan){payload, prefix + length}); + size_t reply = 0; + if (status == PURE_OK) status = pure_app_reply(&app, count_span, &reply); + if (status == PURE_OK && show && reply > budget) { status = PURE_BUDGET; app.revision = 0; } + int exit_code = 0; + if (status != PURE_OK) { + fprintf(stderr, "{\"error\":\"%s\",\"revision\":\"%" PRId64 "\"}\n", message(status), app.revision); + exit_code = status == PURE_CONFLICT ? 3 : 2; + } else if (show) { + if (pure_app_reply(&app, write_span, stdout) != PURE_OK) exit_code = 2; + fprintf(stderr, "{\"revision\":\"%" PRId64 "\",\"bytes\":%zu}\n", app.revision, reply); + } else if (printf("{\"revision\":\"%" PRId64 "\",\"reply_bytes\":%zu}\n", app.revision, reply) < 0) exit_code = 2; + if (fflush(stdout)) exit_code = 2; + pure_app_stop(&app); + free(payload); + return exit_code; +usage: + fputs("pure-edit-module --module FILE --store DIRECTORY [--budget BYTES] show KEY\n" + "pure-edit-module --module FILE --store DIRECTORY [--budget BYTES] edit KEY REVISION BYTE_OFFSET REMOVE_BYTES INSERT_FILE\n" + "pure-edit-module --module FILE --store DIRECTORY [--budget BYTES] apply KEY REVISION INPUT_FILE\n" + "edit uses the module's start and remove fields; apply accepts its raw message bytes.\n", stderr); + return 2; +} diff --git a/cli/pure-run.c b/cli/pure-run.c new file mode 100644 index 0000000..cfc3b5c --- /dev/null +++ b/cli/pure-run.c @@ -0,0 +1,196 @@ +#define _POSIX_C_SOURCE 200809L +#include "module-store.h" +#include "store.h" +#include +#include +#include + +/* Test host for a verified module: reads commands from stdin and prints one + * JSON snapshot per command. Not a product interface. With --store the app's + * states live in that tree store: it starts from them, and frame changes are + * saved at the end of input or by save. --batch N defers saving and saves + * every N commands (and at the end), so N events share one commit. + * down NAME | up NAME | frame [N] | type NAME HEX|- | tick NAME | time MS | replace PATH | save | show + * Text states and text views print as JSON strings; cursors as byte positions; + * lists as arrays of objects; indices as positions; a table view as an array + * with an object of its columns per row. A failed view or cell is null. */ +static const char *const fault_names[] = {"none", "divide", "non_finite", "clamp", "limit", "not_found", "encoding", "exists"}; +/* time MS sets the time events see (milliseconds since 1970); tick NAME is a timer's tick. */ + +static bool load(const char *path, PureModule *module) { + char error[160]; + if (pure_module_read(path, module, error, sizeof(error)) == PURE_OK) return true; + fprintf(stderr, "%s: %s\n", path, error); + return false; +} + +static bool json_span(void *context, PureSpan span) { + (void)context; + for (size_t i = 0; i < span.size; ++i) { + unsigned char c = span.data[i]; + if (c == '"' || c == '\\') printf("\\%c", c); + else if (c < 0x20) printf("\\u%04x", c); + else putchar(c); + } + return true; +} + +/* A text view is collected first: it may fail after some bytes. */ +typedef struct { unsigned char bytes[1 << 16]; size_t used; bool fits; } Collected; + +static bool collect(void *context, PureSpan span) { + Collected *c = context; + c->fits = c->fits && span.size <= sizeof(c->bytes) - c->used; + if (c->fits && span.size) memcpy(c->bytes + c->used, span.data, span.size); + if (c->fits) c->used += span.size; + return true; +} + +static void show_list(const PureModule *m, const PureList *list, const PureRecord *record) { + putchar('['); + for (size_t at = 0; at < pure_list_count(list); ++at) { + printf("%s{", at ? "," : ""); + for (unsigned f = 0; f < record->count; ++f) { + PureCell cell = pure_list_get(list, at, f); + printf("%s\"%s\":", f ? "," : "", m->record_field_names[record->first + f]); + if (!m->record_field_text[record->first + f]) printf("%.17g", cell.num); + else { + putchar('"'); + json_span(NULL, (PureSpan){cell.text, cell.size}); + putchar('"'); + } + } + putchar('}'); + } + putchar(']'); +} + +/* A number or text view, or a column at a row. */ +static void show_value(PureApp *app, uint32_t view, size_t row) { + const PureModule *m = app->module; + static Collected text; + text.used = 0; + text.fits = true; + double value; + bool column = m->view_table[view]; + if (m->view_type[view] == PURE_VIEW_TEXT) { + PureResult result = column ? pure_app_cell_text(app, view, row, collect, &text) + : pure_app_view_text(app, view, collect, &text); + if (result != PURE_OK || !text.fits) printf("null"); + else { + putchar('"'); + json_span(NULL, (PureSpan){text.bytes, text.used}); + putchar('"'); + } + } else if (column ? pure_app_cell(app, view, row, &value) : pure_app_view(app, view, &value)) printf("%.17g", value); + else printf("null"); +} + +static void show(PureApp *app, const char *failure) { + const PureModule *m = app->module; + printf("{\"states\":{"); + for (uint32_t i = 0; i < m->states; ++i) { + printf("%s\"%s\":", i ? "," : "", m->state_names[i]); + if (m->state_type[i] == PURE_STATE_TEXT) { + putchar('"'); + pure_draft_read(app->texts[i], 0, pure_draft_size(app->texts[i]), json_span, NULL); + putchar('"'); + } else if (m->state_type[i] == PURE_STATE_LIST) show_list(m, app->lists[i], &m->record[m->state_ref[i]]); + else printf("%.17g", app->values[i]); + } + printf("},\"views\":{"); + bool first = true; + for (uint32_t i = 0; i < m->views; ++i) { + if (m->view_table[i]) continue; + printf("%s\"%s\":", first ? "" : ",", m->view_names[i]); + first = false; + if (m->view_type[i] != PURE_VIEW_TABLE) { show_value(app, i, 0); continue; } + putchar('['); + for (size_t row = 0; row < pure_app_rows(app, i); ++row) { + printf("%s{", row ? "," : ""); + for (uint32_t c = 0, shown = 0; c < m->views; ++c) { + if (m->view_table[c] != i + 1) continue; + printf("%s\"%s\":", shown++ ? "," : "", m->view_names[c]); + show_value(app, c, row); + } + putchar('}'); + } + putchar(']'); + } + printf("},\"tick\":%llu,\"successfulEvents\":%llu,\"fault\":\"%s\",\"error\":%s%s%s}\n", + (unsigned long long)app->ticks, (unsigned long long)app->events, fault_names[app->fault], + failure ? "\"" : "", failure ? failure : "null", failure ? "\"" : ""); +} + +static int hex(char c) { + return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1; +} + +int main(int argc, char **argv) { + static PureModule modules[2]; + bool stored = argc >= 4 && !strcmp(argv[2], "--store"); + long batch = argc == 6 && stored && !strcmp(argv[4], "--batch") ? strtol(argv[5], NULL, 10) : 0; + bool usage = argc != 2 && !(stored && argc == 4) && !(batch > 0 && batch <= 100000); + if (usage || !load(argv[1], &modules[0])) { + if (usage) fputs("pure-run MODULE [--store PATH [--batch N]] < commands\n", stderr); + return 2; + } + if (stored && pure_store_init(argv[3]) != PURE_OK) { fprintf(stderr, "cannot open store %s\n", argv[3]); return 2; } + unsigned active = 0; + PureApp app; + PureResult started = pure_app_start(&app, &modules[active], stored ? pure_tree_data(argv[3]) : (PureData){0}); + if (started != PURE_OK) { fprintf(stderr, "cannot start (%d)\n", (int)started); return 2; } + if (batch && !pure_app_defer(&app, true)) { fputs("cannot defer saving\n", stderr); return 2; } + long unsaved = 0; + static char line[1 << 16]; + static unsigned char text[1 << 15]; + int status = 0; + while (fgets(line, sizeof(line), stdin)) { + char command[16] = "", argument[256] = "", data[sizeof(line)] = ""; + const char *failure = NULL; + line[strcspn(line, "\r\n")] = 0; + if (sscanf(line, "%15s %255s %65535s", command, argument, data) < 1) continue; + if (!strcmp(command, "down") || !strcmp(command, "up")) { + int input = pure_module_input(app.module, argument); + if (input < 0) failure = "unknown input"; + else if (pure_app_input(&app, (uint32_t)input, command[0] == 'd' ? PURE_EDGE_DOWN : PURE_EDGE_UP) != PURE_OK) + failure = "event failed"; + } else if (!strcmp(command, "type")) { + int input = pure_module_input(app.module, argument); + size_t length = strcmp(data, "-") ? strlen(data) / 2 : 0; + for (size_t i = 0; i < length && length * 2 == strlen(data); ++i) + text[i] = (unsigned char)(hex(data[2 * i]) << 4 | hex(data[2 * i + 1])); + if (input < 0) failure = "unknown input"; + else if (pure_app_text(&app, (uint32_t)input, (PureSpan){text, length}) != PURE_OK) failure = "event failed"; + } else if (!strcmp(command, "tick")) { + int input = pure_module_input(app.module, argument); + if (input < 0) failure = "unknown input"; + else if (pure_app_tick(&app, (uint32_t)input) != PURE_OK) failure = "event failed"; + } else if (!strcmp(command, "time")) { + char *end; + double now = strtod(argument, &end); + if (!*argument || *end || !(now >= 0 && now < 1e15)) failure = "time"; + else app.now = now; + } else if (!strcmp(command, "frame")) { + long count = *argument ? strtol(argument, NULL, 10) : 1; + if (count < 1 || count > 600) failure = "frame count"; + for (long i = 0; !failure && i < count; ++i) + if (pure_app_frame(&app) != PURE_OK) failure = "event failed"; + } else if (!strcmp(command, "replace")) { + if (!load(argument, &modules[!active])) failure = "invalid module"; + else if (pure_app_replace(&app, &modules[!active]) != PURE_OK) failure = "replace failed"; + else active = !active; + } else if (!strcmp(command, "save")) { + if (pure_app_save(&app) != PURE_OK) failure = "save failed"; + } else if (strcmp(command, "show")) failure = "unknown command"; + if (batch && strcmp(command, "show") && ++unsaved == batch) { + unsaved = 0; + if (pure_app_save(&app) != PURE_OK) failure = "save failed"; + } + if (failure) status = 1; + show(&app, failure); + } + if (pure_app_save(&app) != PURE_OK) { fputs("cannot save\n", stderr); status = 1; } + pure_app_stop(&app); + return status; +} diff --git a/core/flow.c b/core/flow.c new file mode 100644 index 0000000..1070c70 --- /dev/null +++ b/core/flow.c @@ -0,0 +1,41 @@ +#include "flow.h" +#include + +void pure_transfer_abort(PureTransfer *t) { + if (!t->active) return; + PureRelease release = t->release; + void *owner = t->owner; + *t = (PureTransfer){0}; + if (release) release(owner); +} + +static void advance(PureTransfer *t) { + while (t->active && t->part < 2 && t->offset == t->spans[t->part].size) { + t->part++; + t->offset = 0; + } + if (t->active && t->part == 2) pure_transfer_abort(t); +} + +void pure_transfer_init(PureTransfer *t, PureSpan header, PureSpan body, + void *owner, PureRelease release) { + assert(!t->active); + assert((!header.size || header.data) && (!body.size || body.data)); + *t = (PureTransfer){.spans = {header, body}, .owner = owner, + .release = release, .active = true}; + advance(t); +} + +PureSpan pure_transfer_next(const PureTransfer *t, size_t limit) { + if (!t->active) return (PureSpan){0}; + PureSpan span = t->spans[t->part]; + size_t remaining = span.size - t->offset; + return (PureSpan){span.data + t->offset, remaining < limit ? remaining : limit}; +} + +bool pure_transfer_consume(PureTransfer *t, size_t bytes) { + if (!t->active || bytes > t->spans[t->part].size - t->offset) return false; + t->offset += bytes; + advance(t); + return true; +} diff --git a/core/flow.h b/core/flow.h new file mode 100644 index 0000000..6e27c58 --- /dev/null +++ b/core/flow.h @@ -0,0 +1,25 @@ +#ifndef PURE_FLOW_H +#define PURE_FLOW_H +#include +#include + +typedef struct { const unsigned char *data; size_t size; } PureSpan; +typedef void (*PureRelease)(void *owner); + +/* A borrowed body stays alive until every byte is consumed or the transfer is + * aborted. A successful send() consumes bytes; peer receipt is a separate fact. + * Single owner thread; callbacks must not reenter or reuse this transfer. */ +typedef struct { + PureSpan spans[2]; + size_t part, offset; + void *owner; + PureRelease release; + bool active; +} PureTransfer; + +void pure_transfer_init(PureTransfer *, PureSpan header, PureSpan body, + void *owner, PureRelease); +PureSpan pure_transfer_next(const PureTransfer *, size_t limit); +bool pure_transfer_consume(PureTransfer *, size_t bytes); +void pure_transfer_abort(PureTransfer *); +#endif diff --git a/core/module-list.c b/core/module-list.c new file mode 100644 index 0000000..2cd2b1c --- /dev/null +++ b/core/module-list.c @@ -0,0 +1,428 @@ +#include "module-list.h" +#include "module-text.h" +#include +#include +#include + +struct PureList { + unsigned fields; + uint8_t text_fields[16]; + PureCell **items; + size_t count, capacity; + PureDraft *draft; /* the stored copy, or NULL */ + size_t header; /* draft bytes before the first item */ + uint64_t id, version; /* which list, and a number every change moves on */ + uint64_t shares; /* copies made so far: an item made before the last one is shared */ + PureRetired *retired; /* where shared items go when the list lets them go */ + unsigned refs; /* a shared copy's references */ + bool copy; /* a shared copy: it owns its item array, not its items */ +}; + +typedef enum { INSERTED, REMOVED, CHANGED, REPLACED } Change; +typedef struct { + Change change; + PureList *list; + size_t at; + unsigned field; + PureCell *item; /* REMOVED: the removed item; REPLACED: the shared item a copy replaced */ + PureCell old; /* CHANGED: the replaced cell */ +} Entry; +typedef struct { PureCell *item; uint16_t text; uint8_t fields; } Retired; + +static uint64_t lists_made; + +/* An item's cells follow the list's share count when the item was made. One + * made before the last share may be read by a shared copy, so it never + * changes again (a change copies it) and is retired instead of freed. */ +static PureCell *new_item(const PureList *list) { + uint64_t *made = calloc(1, sizeof(*made) + (list->fields ? list->fields : 1) * sizeof(PureCell)); + if (!made) return NULL; + *made = list->shares; + return (PureCell *)(made + 1); +} + +static bool shared(const PureList *list, const PureCell *item) { return ((const uint64_t *)item)[-1] < list->shares; } + +static void free_cells(PureCell *item, unsigned fields, unsigned text) { + for (unsigned f = 0; f < fields; ++f) + if (text >> f & 1) free(item[f].text); + free((uint64_t *)item - 1); +} + +static unsigned text_mask(const PureList *list) { + unsigned text = 0; + for (unsigned f = 0; f < list->fields; ++f) text |= (unsigned)(list->text_fields[f] != 0) << f; + return text; +} + +static void free_item(const PureList *list, PureCell *item) { + if (item) free_cells(item, list->fields, text_mask(list)); +} + +/* The list no longer has item: free it, or retire it while a copy may read it. */ +static void let_go(PureList *list, PureCell *item) { + if (!shared(list, item)) { free_item(list, item); return; } + PureRetired *retired = list->retired; + if (retired->count == retired->capacity) { + size_t capacity = retired->capacity ? 2 * retired->capacity : 16; + Retired *grown = realloc(retired->entries, capacity * sizeof(*grown)); + if (!grown) return; /* kept for good rather than freed while read */ + retired->entries = grown; + retired->capacity = capacity; + } + ((Retired *)retired->entries)[retired->count++] = (Retired){item, (uint16_t)text_mask(list), (uint8_t)list->fields}; +} + +void pure_retired_free(PureRetired *retired) { + Retired *entries = retired->entries; + for (size_t i = 0; i < retired->count; ++i) free_cells(entries[i].item, entries[i].fields, entries[i].text); + free(entries); + *retired = (PureRetired){0}; +} + +PureList *pure_list_new(unsigned fields, const uint8_t *text_fields) { + if (fields > 16) return NULL; + PureList *list = calloc(1, sizeof(*list)); + if (!list) return NULL; + list->fields = fields; + memcpy(list->text_fields, text_fields, fields); + list->id = __atomic_add_fetch(&lists_made, 1, __ATOMIC_RELAXED); + return list; +} + +void pure_list_free(PureList *list) { + if (!list) return; + if (list->copy) { + if (__atomic_sub_fetch(&list->refs, 1, __ATOMIC_ACQ_REL)) return; + } else + for (size_t i = 0; i < list->count; ++i) let_go(list, list->items[i]); + free(list->items); + free(list); +} + +size_t pure_list_count(const PureList *list) { return list->count; } + +PureList *pure_list_share(PureList *list, PureRetired *retired) { + PureList *copy = malloc(sizeof(*copy)); + PureCell **items = list->count ? malloc(list->count * sizeof(*items)) : NULL; + if (!copy || (list->count && !items)) { + free(copy); + free(items); + return NULL; + } + if (list->count) memcpy(items, list->items, list->count * sizeof(*items)); + *copy = (PureList){.fields = list->fields, .items = items, .count = list->count, .capacity = list->count, + .id = list->id, .version = list->version, .refs = 1, .copy = true}; + memcpy(copy->text_fields, list->text_fields, sizeof(copy->text_fields)); + list->retired = retired; + ++list->shares; + return copy; +} + +void pure_list_hold(PureList *copy) { __atomic_add_fetch(©->refs, 1, __ATOMIC_RELAXED); } + +bool pure_list_current(const PureList *copy, const PureList *list) { + return copy->id == list->id && copy->version == list->version; +} + +static bool log_change(PureUndo *undo, Entry entry) { + if (undo->count == undo->capacity) { + size_t capacity = undo->capacity ? 2 * undo->capacity : 16; + Entry *entries = realloc(undo->entries, capacity * sizeof(*entries)); + if (!entries) return false; + undo->entries = entries; + undo->capacity = capacity; + } + ((Entry *)undo->entries)[undo->count++] = entry; + return true; +} + +static bool copy_cell(const PureList *list, unsigned field, PureCell from, PureCell *to) { + *to = (PureCell){.num = list->text_fields[field] ? 0 : from.num}; + if (!list->text_fields[field] || !from.size) return true; + to->text = malloc(from.size); + if (!to->text) return false; + memcpy(to->text, from.text, from.size); + to->size = from.size; + return true; +} + +/* ---- The stored form: each field in order, a number as its binary64 bits + * (little-endian), a text as a u32le length and its bytes. */ + +static size_t cell_bytes(const PureList *list, unsigned field, PureCell cell) { + return list->text_fields[field] ? 4 + cell.size : 8; +} + +static size_t item_bytes(const PureList *list, const PureCell *item) { + size_t size = 0; + for (unsigned f = 0; f < list->fields; ++f) size += cell_bytes(list, f, item[f]); + return size; +} + +/* Where an item starts in the draft. Linear in the items before it. */ +static size_t item_offset(const PureList *list, size_t at) { + size_t offset = list->header; + for (size_t i = 0; i < at; ++i) offset += item_bytes(list, list->items[i]); + return offset; +} + +static unsigned char *encode_cell(const PureList *list, unsigned field, PureCell cell, unsigned char *out) { + uint64_t bits = cell.size; + unsigned width = 4; + if (!list->text_fields[field]) { memcpy(&bits, &cell.num, sizeof(bits)); width = 8; } + for (unsigned i = 0; i < width; ++i) *out++ = (unsigned char)(bits >> (8 * i)); + if (list->text_fields[field] && cell.size) { memcpy(out, cell.text, cell.size); out += cell.size; } + return out; +} + +/* Replace remove bytes of the draft, within bytes into the item at `at` (or + * after the last item), with bytes; nothing to do without a draft. */ +static PureResult mirror(PureList *list, size_t at, size_t within, size_t remove, const unsigned char *bytes, + size_t size) { + if (!list->draft) return PURE_OK; + return pure_draft_splice(list->draft, item_offset(list, at) + within, remove, (PureSpan){bytes, size}); +} + +void pure_list_mirror(PureList *list, PureDraft *draft, size_t header) { + list->draft = draft; + list->header = header; +} + +PURE_COLD PureResult pure_list_write(PureList *list) { + size_t size = 0; + for (size_t i = 0; i < list->count; ++i) size += item_bytes(list, list->items[i]); + unsigned char *bytes = size ? malloc(size) : NULL, *at = bytes; + if (size && !bytes) return PURE_ALLOCATION; + for (size_t i = 0; i < list->count; ++i) + for (unsigned f = 0; f < list->fields; ++f) at = encode_cell(list, f, list->items[i][f], at); + PureResult result = mirror(list, 0, 0, 0, bytes, size); + free(bytes); + return result; +} + +PURE_COLD PureResult pure_list_read(PureList *list, const unsigned char *bytes, size_t size) { + PureCell cells[16]; + for (size_t at = 0; at < size;) { + for (unsigned f = 0; f < list->fields; ++f) { + size_t width = list->text_fields[f] ? 4 : 8; + if (size - at < width) return PURE_STORE_ERROR; + uint64_t bits = 0; + for (size_t i = 0; i < width; ++i) bits |= (uint64_t)bytes[at + i] << (8 * i); + at += width; + if (!list->text_fields[f]) { + cells[f] = (PureCell){0}; + memcpy(&cells[f].num, &bits, sizeof(bits)); + if (!isfinite(cells[f].num)) return PURE_STORE_ERROR; + continue; + } + if (bits > size - at || !pure_utf8_valid((PureSpan){bytes + at, (size_t)bits})) + return PURE_STORE_ERROR; + cells[f] = (PureCell){0, (unsigned char *)bytes + at, (size_t)bits}; + at += (size_t)bits; + } + PureResult result = pure_list_insert(list, list->count, cells, NULL); + if (result != PURE_OK) return result; + } + return PURE_OK; +} + +static void place(PureList *list, size_t at, PureCell *item) { + memmove(list->items + at + 1, list->items + at, (list->count - at) * sizeof(*list->items)); + list->items[at] = item; + ++list->count; + ++list->version; +} + +static PureCell *take(PureList *list, size_t at) { + PureCell *item = list->items[at]; + memmove(list->items + at, list->items + at + 1, (list->count - at - 1) * sizeof(*list->items)); + --list->count; + ++list->version; + return item; +} + +PureResult pure_list_insert(PureList *list, size_t at, const PureCell *cells, PureUndo *undo) { + if (at > list->count || list->count == PURE_LIST_ITEMS) return PURE_LIMIT; + if (list->count == list->capacity) { + size_t capacity = list->capacity ? 2 * list->capacity : 8; + PureCell **items = realloc(list->items, capacity * sizeof(*items)); + if (!items) return PURE_ALLOCATION; + list->items = items; + list->capacity = capacity; + } + PureCell *item = new_item(list); + if (!item) return PURE_ALLOCATION; + for (unsigned f = 0; f < list->fields; ++f) + if (!copy_cell(list, f, cells[f], &item[f])) { free_item(list, item); return PURE_ALLOCATION; } + size_t size = list->draft ? item_bytes(list, item) : 0; + unsigned char *bytes = size ? malloc(size) : NULL, *end = bytes; + if (size && !bytes) { free_item(list, item); return PURE_ALLOCATION; } + for (unsigned f = 0; bytes && f < list->fields; ++f) end = encode_cell(list, f, item[f], end); + if (undo && !log_change(undo, (Entry){.change = INSERTED, .list = list, .at = at})) { + free(bytes); + free_item(list, item); + return PURE_ALLOCATION; + } + PureResult result = mirror(list, at, 0, 0, bytes, size); + free(bytes); + if (result != PURE_OK) { + if (undo) --undo->count; + free_item(list, item); + return result; + } + place(list, at, item); + return PURE_OK; +} + +PureResult pure_list_remove(PureList *list, size_t at, PureUndo *undo) { + if (at >= list->count) return PURE_NOT_FOUND; + if (!log_change(undo, (Entry){.change = REMOVED, .list = list, .at = at, .item = list->items[at]})) + return PURE_ALLOCATION; + PureResult result = mirror(list, at, 0, item_bytes(list, list->items[at]), NULL, 0); + if (result != PURE_OK) { --undo->count; return result; } + take(list, at); + return PURE_OK; +} + +/* A shared item's copy with every field but one (left empty). */ +static PureCell *copy_item(const PureList *list, const PureCell *item, unsigned except) { + PureCell *fresh = new_item(list); + for (unsigned f = 0; fresh && f < list->fields; ++f) + if (f != except && !copy_cell(list, f, item[f], &fresh[f])) { + free_item(list, fresh); + fresh = NULL; + } + return fresh; +} + +PureResult pure_list_set(PureList *list, size_t at, unsigned field, PureCell value, PureUndo *undo) { + if (at >= list->count) return PURE_NOT_FOUND; + PureCell copy, *item = list->items[at]; + if (!copy_cell(list, field, value, ©)) return PURE_ALLOCATION; + PureCell *fresh = shared(list, item) ? copy_item(list, item, field) : item; + size_t size = list->draft ? cell_bytes(list, field, copy) : 0; + unsigned char *bytes = size ? malloc(size) : NULL; + Entry entry = fresh == item ? (Entry){.change = CHANGED, .list = list, .at = at, .field = field, .old = item[field]} : + (Entry){.change = REPLACED, .list = list, .at = at, .item = item}; + if (!fresh || (size && !bytes) || !log_change(undo, entry)) { + free(bytes); + free(copy.text); + if (fresh != item) free_item(list, fresh); + return PURE_ALLOCATION; + } + if (bytes) encode_cell(list, field, copy, bytes); + size_t within = 0; + for (unsigned f = 0; f < field; ++f) within += cell_bytes(list, f, item[f]); + PureResult result = mirror(list, at, within, cell_bytes(list, field, item[field]), bytes, size); + free(bytes); + if (result != PURE_OK) { + --undo->count; + free(copy.text); + if (fresh != item) free_item(list, fresh); + return result; + } + fresh[field] = copy; + list->items[at] = fresh; + ++list->version; + return PURE_OK; +} + +PureCell pure_list_get(const PureList *list, size_t at, unsigned field) { + return list->items[at][field]; +} + +/* How two cells of a field order: numbers by value (-0 and +0 are one key), + * texts by their bytes. */ +static int order(const PureList *list, unsigned field, PureCell a, PureCell b) { + if (!list->text_fields[field]) return (a.num > b.num) - (a.num < b.num); + size_t shorter = a.size < b.size ? a.size : b.size; + int bytes = shorter ? memcmp(a.text, b.text, shorter) : 0; + return bytes ? (bytes > 0) - (bytes < 0) : (a.size > b.size) - (a.size < b.size); +} + +size_t pure_list_seek(const PureList *list, unsigned field, PureCell key, bool *found) { + size_t low = 0, high = list->count; + while (low < high) { + size_t middle = low + (high - low) / 2; + if (order(list, field, list->items[middle][field], key) < 0) low = middle + 1; + else high = middle; + } + *found = low < list->count && !order(list, field, list->items[low][field], key); + return low; +} + +/* Positions sorted by their items' field, stably (spare holds count). */ +PURE_COLD static void merge_sort(const PureList *list, unsigned field, size_t *at, size_t *spare, size_t count) { + if (count < 2) return; + size_t half = count / 2, i = 0, j = half, k = 0; + merge_sort(list, field, at, spare, half); + merge_sort(list, field, at + half, spare, count - half); + while (i < half && j < count) + spare[k++] = order(list, field, list->items[at[j]][field], list->items[at[i]][field]) < 0 ? at[j++] : at[i++]; + while (i < half) spare[k++] = at[i++]; + while (j < count) spare[k++] = at[j++]; + memcpy(at, spare, count * sizeof(*at)); +} + +PURE_COLD PureResult pure_list_sorted(const PureList *list, unsigned field, PureList **out, size_t *moved) { + *out = NULL; + bool ordered = true; + for (size_t i = 1; i < list->count && ordered; ++i) + ordered = order(list, field, list->items[i - 1][field], list->items[i][field]) < 0; + if (ordered) return PURE_OK; + size_t *at = malloc(2 * list->count * sizeof(*at)); + if (!at) return PURE_ALLOCATION; + for (size_t i = 0; i < list->count; ++i) at[i] = i; + merge_sort(list, field, at, at + list->count, list->count); + PureResult result = PURE_OK; + for (size_t i = 1; i < list->count && result == PURE_OK; ++i) + if (!order(list, field, list->items[at[i - 1]][field], list->items[at[i]][field])) result = PURE_CONFLICT; + PureList *sorted = result == PURE_OK ? pure_list_new(list->fields, list->text_fields) : NULL; + if (result == PURE_OK && !sorted) result = PURE_ALLOCATION; + for (size_t i = 0; result == PURE_OK && i < list->count; ++i) { + result = pure_list_insert(sorted, i, list->items[at[i]], NULL); + moved[at[i]] = i; + } + free(at); + if (result != PURE_OK) { + pure_list_free(sorted); + return result; + } + *out = sorted; + return PURE_OK; +} + +void pure_undo_rollback(PureUndo *undo, size_t mark) { + Entry *entries = undo->entries; + for (size_t i = undo->count; i-- > mark;) { + Entry *e = &entries[i]; + if (e->change == INSERTED) free_item(e->list, take(e->list, e->at)); + else if (e->change == REMOVED) place(e->list, e->at, e->item); /* capacity is still there */ + else if (e->change == REPLACED) { + free_item(e->list, e->list->items[e->at]); + e->list->items[e->at] = e->item; + ++e->list->version; + } else { + if (e->list->text_fields[e->field]) free(e->list->items[e->at][e->field].text); + e->list->items[e->at][e->field] = e->old; + ++e->list->version; + } + } + undo->count = mark; +} + +void pure_undo_commit(PureUndo *undo) { + Entry *entries = undo->entries; + for (size_t i = 0; i < undo->count; ++i) { + Entry *e = &entries[i]; + if (e->change == REMOVED || e->change == REPLACED) let_go(e->list, e->item); + else if (e->change == CHANGED && e->list->text_fields[e->field]) free(e->old.text); + } + undo->count = 0; +} + +void pure_undo_free(PureUndo *undo) { + free(undo->entries); + *undo = (PureUndo){0}; +} diff --git a/core/module-list.h b/core/module-list.h new file mode 100644 index 0000000..a721ff0 --- /dev/null +++ b/core/module-list.h @@ -0,0 +1,74 @@ +#ifndef PURE_MODULE_LIST_H +#define PURE_MODULE_LIST_H +#include "result.h" +#include "state.h" +#include +#include +#include + +/* Executor paths that run once per module or app (loading, verifying, + * starting, replacing, reading stored states) are built for size; events + * stay built for speed. */ +#if defined(__GNUC__) || defined(__clang__) +#define PURE_COLD __attribute__((cold)) +#else +#define PURE_COLD +#endif + +/* Runtime-owned lists of records for the module executor. An item is one + * cell per field: a number, or text bytes the list owns. Every change made + * during an event is logged; a failed event undoes the log in reverse and a + * successful one frees what it replaced. */ +typedef struct { double num; unsigned char *text; size_t size; } PureCell; +typedef struct PureList PureList; +typedef struct { void *entries; size_t count, capacity; } PureUndo; +/* Items a list let go while a shared copy may still read them. */ +typedef struct { void *entries; size_t count, capacity; } PureRetired; + +enum { PURE_LIST_ITEMS = 65536 }; + +PureList *pure_list_new(unsigned fields, const uint8_t *text_fields); +void pure_list_free(PureList *); +size_t pure_list_count(const PureList *); +/* Share the items as they are with a read-only copy (for readers on other + * threads), without copying them: from then on the list copies an item it + * changes, and items it lets go go to retired instead of being freed, since + * the copy may still read them. The copy is freed with pure_list_free, on any + * thread, when its last reference goes (pure_list_hold takes one more). Only + * between events (no change waits to be committed or undone). */ +PureList *pure_list_share(PureList *, PureRetired *retired); +void pure_list_hold(PureList *copy); +/* Whether copy shares list's items as they are now. */ +bool pure_list_current(const PureList *copy, const PureList *list); +/* Free items retired while no copy that shared them remains. */ +void pure_retired_free(PureRetired *); +/* Insert a copy of cells (text bytes are copied) before position at. A NULL + * log means the change is not undone. */ +PureResult pure_list_insert(PureList *, size_t at, const PureCell *cells, PureUndo *); +PureResult pure_list_remove(PureList *, size_t at, PureUndo *); +PureResult pure_list_set(PureList *, size_t at, unsigned field, PureCell value, PureUndo *); +PureCell pure_list_get(const PureList *, size_t at, unsigned field); +/* In a list ordered by field (a keyed list): the first position whose item's + * field does not order below key (numbers by value, texts by bytes), and + * whether that item's field is key. */ +size_t pure_list_seek(const PureList *, unsigned field, PureCell key, bool *found); +/* The list's items ordered by field, as a keyed list keeps them: *out is NULL + * when they already are, or a new list in that order with moved[position + * before] = position after for every item. CONFLICT when two items have the + * same key (nothing is made). */ +PureResult pure_list_sorted(const PureList *, unsigned field, PureList **out, size_t *moved); +/* A list may keep a stored copy in a draft: header bytes, then each item's + * fields in order (a number as its binary64 bits, little-endian; a text as a + * u32le length and its bytes). Every change splices the draft as well, and a + * change the draft refuses (such as its size limit) does not happen. Undoing + * an event changes only the list: the caller restores the draft's version. */ +void pure_list_mirror(PureList *, PureDraft *, size_t header); +/* Store every item after the header (the draft holds just the header). */ +PureResult pure_list_write(PureList *); +/* Append the items stored in bytes; each field is checked. Before mirroring. */ +PureResult pure_list_read(PureList *, const unsigned char *bytes, size_t size); +/* Undo the changes logged after mark (a count taken earlier), newest first. */ +void pure_undo_rollback(PureUndo *, size_t mark); +void pure_undo_commit(PureUndo *); +void pure_undo_free(PureUndo *); +#endif diff --git a/core/module-save.c b/core/module-save.c new file mode 100644 index 0000000..4149316 --- /dev/null +++ b/core/module-save.c @@ -0,0 +1,488 @@ +#include "module-save.h" +#include "module-text.h" +#include +#include +#include +#include + +/* Stored forms. values: "PVS1", then per state {u8 type, u8 name length, + * name, binary64 bits (little-endian)}. A list: "PLS1", u8 field count, per + * field {u8 0 number | 1 text, u8 name length, name}, then its items + * (module-list.h). A text state: its UTF-8 bytes. */ +static const unsigned char values_magic[4] = {'P', 'V', 'S', '1'}, list_magic[4] = {'P', 'L', 'S', '1'}; +/* links: "PLK1"; u8 outputs, each {u8 name length, name, u8 field count, the + * fields' types, u64le last number} (the module's outputs, then those it + * dropped, without fields); u16le sources, each {u8 length, the name, u64le + * last number heard}; then the sent messages to the end, each {u8 output (in + * the table), u64le number, u32le size, payload}. */ +static const unsigned char links_magic[4] = {'P', 'L', 'K', '1'}; +static const PureVersion unsaved = {UINT64_MAX, UINT64_MAX}; + +bool pure_save_enabled(const PureApp *app) { return app->data.state && app->data.save; } + +static PureResult open_state(PureApp *app, const char *prefix, const char *name, PureDraft **out) { + char key[64]; + snprintf(key, sizeof(key), "%s%s", prefix, name); + return app->data.state(app->data.context, key, out); +} + +/* The whole value of a draft, contiguous; free it. */ +typedef struct { unsigned char *bytes; size_t used; } Whole; + +static bool gather(void *context, PureSpan span) { + Whole *w = context; + memcpy(w->bytes + w->used, span.data, span.size); + w->used += span.size; + return true; +} + +static PureResult read_whole(const PureDraft *draft, unsigned char **out, size_t *size) { + *size = pure_draft_size(draft); + Whole w = {malloc(*size ? *size : 1), 0}; + if (!w.bytes) return PURE_ALLOCATION; + PureResult result = pure_draft_read(draft, 0, *size, gather, &w); + if (result != PURE_OK) { free(w.bytes); return result; } + *out = w.bytes; + return PURE_OK; +} + +/* One named entry of a stored form: u8 kind, u8 name length, the name (an + * identifier), then width more bytes. */ +PURE_COLD static bool named(const unsigned char *bytes, size_t size, size_t *at, size_t width, uint8_t *kind, + char name[PURE_MODULE_NAME]) { + if (size - *at < 2) return false; + size_t length = bytes[*at + 1]; + if (size - *at - 2 < length + width || !length || length >= PURE_MODULE_NAME) return false; + for (size_t i = 0; i < length; ++i) { + unsigned char c = bytes[*at + 2 + i]; + bool letter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; + if (!letter && !(i && c >= '0' && c <= '9')) return false; + } + *kind = bytes[*at]; + memcpy(name, bytes + *at + 2, length); + name[length] = 0; + *at += 2 + length; + return true; +} + +PURE_COLD PureResult pure_save_open_text(PureApp *app, const PureModule *m, uint32_t state, PureDraft **out, + PureVersion *saved) { + PureResult result = open_state(app, "text-", m->state_names[state], out); + if (result != PURE_OK) return result; + *saved = pure_draft_version(*out); + if (!pure_draft_revision(*out)) { + uint32_t constant = m->state_ref[state]; + PureSpan initial = {m->constant_data + m->constant_offset[constant], m->constant_length[constant]}; + result = pure_draft_splice(*out, 0, pure_draft_size(*out), initial); + *saved = unsaved; + } else { + unsigned char *bytes; + size_t size; + result = read_whole(*out, &bytes, &size); + if (result == PURE_OK) { + if (!pure_utf8_valid((PureSpan){bytes, size})) result = PURE_STORE_ERROR; + free(bytes); + } + } + if (result != PURE_OK) { pure_draft_close(*out); *out = NULL; } + return result; +} + +static size_t list_header(const PureModule *m, uint32_t record, unsigned char *out) { + const PureRecord *r = &m->record[record]; + size_t at = 0; + memcpy(out, list_magic, 4); + at = 4; + out[at++] = (unsigned char)r->count; + for (unsigned f = 0; f < r->count; ++f) { + const char *name = m->record_field_names[r->first + f]; + out[at++] = m->record_field_text[r->first + f]; + out[at++] = (unsigned char)strlen(name); + memcpy(out + at, name, strlen(name)); + at += strlen(name); + } + return at; +} + +PURE_COLD PureResult pure_save_rewrite_list(const PureModule *m, uint32_t state, PureList *list, PureDraft *draft) { + unsigned char header[5 + PURE_RECORD_FIELDS * (2 + PURE_MODULE_NAME)]; + size_t length = list_header(m, m->state_ref[state], header); + PureVersion before = pure_draft_version(draft); + pure_draft_hold(draft, true); + PureResult result = pure_draft_splice(draft, 0, pure_draft_size(draft), (PureSpan){header, length}); + pure_list_mirror(list, draft, length); + if (result == PURE_OK) result = pure_list_write(list); + if (result != PURE_OK) { + pure_draft_restore(draft, before); + pure_list_mirror(list, NULL, 0); + } + pure_draft_hold(draft, false); + return result; +} + +PURE_COLD PureResult pure_save_open_list(PureApp *app, const PureModule *m, uint32_t state, PureList **out, PureDraft **draft, + PureVersion *saved) { + uint32_t record = m->state_ref[state]; + const PureRecord *r = &m->record[record]; + *out = NULL; + PureResult result = open_state(app, "list-", m->state_names[state], draft); + if (result != PURE_OK) return result; + unsigned char *bytes = NULL; + size_t size = 0; + if (pure_draft_revision(*draft)) result = read_whole(*draft, &bytes, &size); + /* The stored record: its fields' kinds and names. */ + char names[PURE_RECORD_FIELDS][PURE_MODULE_NAME] = {{0}}; + uint8_t text[PURE_RECORD_FIELDS] = {0}; + unsigned count = 0; + size_t at = 0; + bool stored = result == PURE_OK && bytes; + if (stored) { + bool ok = size >= 5 && !memcmp(bytes, list_magic, 4) && bytes[4] <= PURE_RECORD_FIELDS; + count = ok ? bytes[4] : 0; + at = 5; + for (unsigned f = 0; ok && f < count; ++f) { + ok = named(bytes, size, &at, 0, &text[f], names[f]) && text[f] <= 1; + for (unsigned k = 0; k < f; ++k) ok = ok && strcmp(names[k], names[f]); + } + if (!ok) result = PURE_STORE_ERROR; + } + PureList *from = NULL; + if (result == PURE_OK && stored) { + from = pure_list_new(count, text); + result = from ? pure_list_read(from, bytes + at, size - at) : PURE_ALLOCATION; + } + free(bytes); + bool same = stored && count == r->count; + for (unsigned f = 0; same && f < count; ++f) + same = text[f] == m->record_field_text[r->first + f] && !strcmp(names[f], m->record_field_names[r->first + f]); + if (result == PURE_OK && same) { + *out = from; + pure_list_mirror(from, *draft, at); + *saved = pure_draft_version(*draft); + return PURE_OK; + } + if (result == PURE_OK) { + /* New, or stored with another record: its items move by field name. */ + const char *field_names[PURE_RECORD_FIELDS]; + for (unsigned f = 0; f < count; ++f) field_names[f] = names[f]; + if (from) result = pure_list_migrate(out, from, field_names, text, count, m, record); + else if (!(*out = pure_list_new(r->count, m->record_field_text + r->first))) result = PURE_ALLOCATION; + } + pure_list_free(from); + if (result == PURE_OK) result = pure_save_rewrite_list(m, state, *out, *draft); + if (result != PURE_OK) { + pure_list_free(*out); + pure_draft_close(*draft); + *out = NULL; + *draft = NULL; + return result; + } + *saved = unsaved; + return PURE_OK; +} + +static bool scalar(uint8_t type) { + return type == PURE_STATE_NUM || type == PURE_STATE_CURSOR || type == PURE_STATE_INDEX; +} + +/* A stored value still fits its state: a cursor on a character boundary of + * its text, an index on an item of its list (or 0), a finite number. */ +PURE_COLD static bool fits(const PureModule *m, PureDraft *const *texts, PureList *const *lists, uint32_t state, double value) { + if (!isfinite(value)) return false; + if (m->state_type[state] == PURE_STATE_NUM) return true; + if (value < 0 || value != floor(value)) return false; + if (m->state_type[state] == PURE_STATE_INDEX) { + size_t count = pure_list_count(lists[m->state_ref[state]]); + return value <= (count ? (double)(count - 1) : 0); + } + const PureDraft *text = texts[m->state_ref[state]]; + size_t size = pure_draft_size(text), at = (size_t)value; + if (value > (double)size) return false; + if (at == size) return true; + unsigned char byte = 0; + Whole w = {&byte, 0}; + return pure_draft_read(text, at, 1, gather, &w) == PURE_OK && (byte & 0xC0) != 0x80; +} + +static size_t encode_values(const PureModule *m, const double *values, unsigned char *out) { + size_t at = 4; + memcpy(out, values_magic, 4); + for (uint32_t i = 0; i < m->states; ++i) { + if (!scalar(m->state_type[i])) continue; + size_t length = strlen(m->state_names[i]); + uint64_t bits; + memcpy(&bits, &values[i], sizeof(bits)); + out[at++] = m->state_type[i]; + out[at++] = (unsigned char)length; + memcpy(out + at, m->state_names[i], length); + at += length; + for (unsigned b = 0; b < 8; ++b) out[at++] = (unsigned char)(bits >> (8 * b)); + } + return at; +} + +enum { VALUES_BYTES = 4 + PURE_MODULE_STATES * (2 + PURE_MODULE_NAME + 8) }; + +PURE_COLD PureResult pure_save_open_values(PureApp *app, const PureModule *m, double *values, const int *kept, + PureDraft *const *texts, PureList *const *lists) { + bool scalars = false, fresh = false; + for (uint32_t i = 0; i < m->states; ++i) + if (scalar(m->state_type[i])) { + scalars = true; + fresh = fresh || kept[i] < 0; + } + if (!scalars) return PURE_OK; + PureResult result = app->stored_values ? PURE_OK : open_state(app, "", "values", &app->stored_values); + if (result != PURE_OK || !fresh || !pure_draft_revision(app->stored_values)) return result; + unsigned char *bytes; + size_t size; + if ((result = read_whole(app->stored_values, &bytes, &size)) != PURE_OK) return result; + bool ok = size >= 4 && !memcmp(bytes, values_magic, 4); + for (size_t at = 4; ok && at < size;) { + char name[PURE_MODULE_NAME]; + uint8_t type; + if (!(ok = named(bytes, size, &at, 8, &type, name))) break; + uint64_t bits = 0; + for (unsigned b = 0; b < 8; ++b) bits |= (uint64_t)bytes[at + b] << (8 * b); + at += 8; + double value; + memcpy(&value, &bits, sizeof(value)); + int state = pure_module_state(m, name); + if (state >= 0 && kept[state] < 0 && m->state_type[state] == type && scalar(type) && + fits(m, texts, lists, (uint32_t)state, value)) + values[state] = value; + } + free(bytes); + return ok ? PURE_OK : PURE_STORE_ERROR; +} + +static bool same_version(PureVersion a, PureVersion b) { return a.ref == b.ref && a.bytes == b.bytes; } + +static unsigned char *put_le(unsigned char *at, uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) *at++ = (unsigned char)(value >> (8 * i)); + return at; +} + +static uint64_t get_le(const unsigned char *at, unsigned width) { + uint64_t value = 0; + for (unsigned i = 0; i < width; ++i) value |= (uint64_t)at[i] << (8 * i); + return value; +} + +/* The links as stored; NULL when memory runs out. */ +static unsigned char *encode_links(const PureApp *app, size_t *size) { + const PureModule *m = app->module; + unsigned outputs = 0; + uint8_t table[PURE_MODULE_INPUTS]; + size_t total = 4 + 1 + 2; + for (uint32_t i = 0; i < m->inputs; ++i) + if (m->input[i].kind == PURE_KIND_OUTPUT) { + table[i] = (uint8_t)outputs++; + total += 2 + strlen(m->input_names[i]) + m->input[i].field_count + 8; + } + for (unsigned c = 0; c < app->carried_count; ++c) total += 2 + strlen(app->carried[c].name) + 8; + for (unsigned h = 0; h < app->heard_count; ++h) total += 1 + strlen(app->heard[h].source) + 8; + for (size_t s = 0; s < app->sent_count; ++s) + if (!app->sent[s].trimmed) total += 1 + 8 + 4 + app->sent[s].size; + unsigned char *bytes = malloc(total), *at = bytes; + if (!bytes) return NULL; + memcpy(at, links_magic, 4); + at += 4; + *at++ = (unsigned char)(outputs + app->carried_count); + for (uint32_t i = 0; i < m->inputs; ++i) { + if (m->input[i].kind != PURE_KIND_OUTPUT) continue; + size_t length = strlen(m->input_names[i]); + *at++ = (unsigned char)length; + memcpy(at, m->input_names[i], length); + at += length; + *at++ = (unsigned char)m->input[i].field_count; + for (uint32_t f = 0; f < m->input[i].field_count; ++f) *at++ = m->field[m->input[i].field_start + f].type; + at = put_le(at, app->last_sent[i], 8); + } + for (unsigned c = 0; c < app->carried_count; ++c) { + size_t length = strlen(app->carried[c].name); + *at++ = (unsigned char)length; + memcpy(at, app->carried[c].name, length); + at += length; + *at++ = 0; + at = put_le(at, app->carried[c].last, 8); + } + at = put_le(at, app->heard_count, 2); + for (unsigned h = 0; h < app->heard_count; ++h) { + size_t length = strlen(app->heard[h].source); + *at++ = (unsigned char)length; + memcpy(at, app->heard[h].source, length); + at = put_le(at + length, app->heard[h].seq, 8); + } + for (size_t s = 0; s < app->sent_count; ++s) { + const struct PureSent *sent = &app->sent[s]; + if (sent->trimmed) continue; + *at++ = table[sent->output]; + at = put_le(at, sent->seq, 8); + at = put_le(at, sent->size, 4); + if (sent->size) memcpy(at, sent->payload, sent->size); + at += sent->size; + } + *size = (size_t)(at - bytes); + return bytes; +} + +/* The stored links of a starting app: outputs by name (an output whose + * fields changed or that the module lacks refuses the start while it has + * sends left: CONFLICT; one without sends keeps its number for later), + * sources and sent messages. */ +PURE_COLD PureResult pure_save_open_links(PureApp *app, const PureModule *m) { + PureResult result = app->stored_links ? PURE_OK : open_state(app, "", "links", &app->stored_links); + if (result != PURE_OK || !pure_draft_revision(app->stored_links)) return result; + unsigned char *bytes; + size_t size; + if ((result = read_whole(app->stored_links, &bytes, &size)) != PURE_OK) return result; + int output[256]; + bool fits[256]; + size_t at = 5; + unsigned outputs = size >= 5 && !memcmp(bytes, links_magic, 4) ? bytes[4] : 0; + result = size >= 5 && !memcmp(bytes, links_magic, 4) ? PURE_OK : PURE_STORE_ERROR; + for (unsigned t = 0; t < outputs && result == PURE_OK; ++t) { + char name[PURE_MODULE_NAME]; + size_t length = at < size ? bytes[at] : 0; + if (!length || length >= PURE_MODULE_NAME || size - at < 2 + length) { result = PURE_STORE_ERROR; break; } + memcpy(name, bytes + at + 1, length); + name[length] = 0; + unsigned fields = bytes[at + 1 + length]; + at += 2 + length; + if (size - at < fields + 8) { result = PURE_STORE_ERROR; break; } + int i = pure_module_input(m, name); + output[t] = i >= 0 && m->input[i].kind == PURE_KIND_OUTPUT ? i : -1; + fits[t] = output[t] >= 0 && m->input[i].field_count == fields; + for (unsigned f = 0; fits[t] && f < fields; ++f) fits[t] = m->field[m->input[i].field_start + f].type == bytes[at + f]; + uint64_t last = get_le(bytes + at + fields, 8); + at += fields + 8; + if (output[t] >= 0) app->last_sent[output[t]] = last; + else if (last) { + struct PureCarried *grown = realloc(app->carried, (app->carried_count + 1) * sizeof(*grown)); + if (!grown) { result = PURE_ALLOCATION; break; } + app->carried = grown; + memcpy(grown[app->carried_count].name, name, length + 1); + grown[app->carried_count++].last = last; + } + } + unsigned sources = result == PURE_OK && size - at >= 2 ? (unsigned)get_le(bytes + at, 2) : 0; + if (result == PURE_OK && size - at < 2) result = PURE_STORE_ERROR; + at += 2; + if (result == PURE_OK && sources && !(app->heard = calloc(sources, sizeof(*app->heard)))) result = PURE_ALLOCATION; + app->heard_capacity = sources; + for (unsigned h = 0; h < sources && result == PURE_OK; ++h) { + size_t length = at < size ? bytes[at] : 0; + if (!length || length > PURE_SOURCE || size - at < 1 + length + 8) { result = PURE_STORE_ERROR; break; } + struct PureHeard *heard = &app->heard[app->heard_count++]; + memcpy(heard->source, bytes + at + 1, length); + heard->seq = heard->saved = get_le(bytes + at + 1 + length, 8); + at += 1 + length + 8; + } + while (result == PURE_OK && at < size) { + if (size - at < 13 || bytes[at] >= outputs) { result = PURE_STORE_ERROR; break; } + unsigned t = bytes[at]; + uint64_t seq = get_le(bytes + at + 1, 8); + size_t length = (size_t)get_le(bytes + at + 9, 4); + at += 13; + if (size - at < length) { result = PURE_STORE_ERROR; break; } + if (!fits[t]) { result = PURE_CONFLICT; break; } + if (app->sent_count == app->sent_capacity) { + size_t capacity = app->sent_capacity ? 2 * app->sent_capacity : 16; + struct PureSent *grown = realloc(app->sent, capacity * sizeof(*grown)); + if (!grown) { result = PURE_ALLOCATION; break; } + app->sent = grown; + app->sent_capacity = capacity; + } + unsigned char *payload = malloc(length ? length : 1); + if (!payload) { result = PURE_ALLOCATION; break; } + if (length) memcpy(payload, bytes + at, length); + app->sent[app->sent_count++] = (struct PureSent){(uint32_t)output[t], seq, payload, length, false}; + at += length; + } + free(bytes); + app->sent_durable = app->sent_count; + memcpy(app->saved_sent, app->last_sent, sizeof(app->saved_sent)); + return result; +} + +PureResult pure_save_commit(PureApp *app, const double *values, void *const *documents, size_t document_count, + int64_t *revisions) { + if (!pure_save_enabled(app)) { + /* Without a host that saves together, a save has one document at most. */ + PureResult result = PURE_OK; + for (size_t i = 0; i < document_count && result == PURE_OK; ++i) + result = app->data.commit(app->data.context, documents[i], &revisions[i]); + return result; + } + const PureModule *m = app->module; + PureDraft *drafts[2 * PURE_MODULE_STATES + 2]; + uint32_t states[2 * PURE_MODULE_STATES + 2]; + size_t count = 0; + for (uint32_t i = 0; i < m->states; ++i) { + PureDraft *draft = m->state_type[i] == PURE_STATE_TEXT ? app->texts[i] : + m->state_type[i] == PURE_STATE_LIST ? app->list_drafts[i] : NULL; + if (draft && !same_version(pure_draft_version(draft), app->saved[i])) { + states[count] = i; + drafts[count++] = draft; + } + } + /* Numbers, cursors and indices: stored again only when their bytes change. */ + PureVersion values_before = {0, 0}; + bool values_changed = false; + if (app->stored_values) { + unsigned char encoded[VALUES_BYTES], *stored; + size_t length = encode_values(m, values, encoded), size; + PureResult result = read_whole(app->stored_values, &stored, &size); + if (result != PURE_OK) return result; + values_changed = size != length || memcmp(stored, encoded, length); + free(stored); + if (values_changed) { + values_before = pure_draft_version(app->stored_values); + pure_draft_hold(app->stored_values, true); + result = pure_draft_splice(app->stored_values, 0, size, (PureSpan){encoded, length}); + if (result != PURE_OK) { pure_draft_hold(app->stored_values, false); return result; } + states[count] = UINT32_MAX; + drafts[count++] = app->stored_values; + } + } + /* The links, written whole when they changed. */ + PureVersion links_before = {0, 0}; + bool links_written = false; + if (app->links_changed) { + PureResult result = app->stored_links ? PURE_OK : open_state(app, "", "links", &app->stored_links); + size_t length; + unsigned char *encoded = result == PURE_OK ? encode_links(app, &length) : NULL; + if (result == PURE_OK && !encoded) result = PURE_ALLOCATION; + if (result == PURE_OK) { + links_before = pure_draft_version(app->stored_links); + pure_draft_hold(app->stored_links, true); + result = pure_draft_splice(app->stored_links, 0, pure_draft_size(app->stored_links), (PureSpan){encoded, length}); + if (result != PURE_OK) pure_draft_hold(app->stored_links, false); + } + free(encoded); + if (result != PURE_OK) { + if (values_changed) { + pure_draft_restore(app->stored_values, values_before); + pure_draft_hold(app->stored_values, false); + } + return result; + } + links_written = true; + states[count] = UINT32_MAX; + drafts[count++] = app->stored_links; + } + if (!count && !document_count) return PURE_OK; + PureResult result = app->data.save(app->data.context, documents, document_count, drafts, count, revisions); + for (size_t k = 0; result == PURE_OK && k < count; ++k) + if (states[k] != UINT32_MAX) app->saved[states[k]] = pure_draft_version(drafts[k]); + if (values_changed) { + if (result != PURE_OK) pure_draft_restore(app->stored_values, values_before); + pure_draft_hold(app->stored_values, false); + } + if (links_written) { + if (result != PURE_OK) pure_draft_restore(app->stored_links, links_before); + pure_draft_hold(app->stored_links, false); + } + return result; +} diff --git a/core/module-save.h b/core/module-save.h new file mode 100644 index 0000000..bc7adcc --- /dev/null +++ b/core/module-save.h @@ -0,0 +1,36 @@ +#ifndef PURE_MODULE_SAVE_H +#define PURE_MODULE_SAVE_H +#include "module.h" + +/* App states a data host keeps (docs/design/008-module-format.md), internal + * to the module executor. Each text state is a stored draft, each list state + * a stored draft its list mirrors, and numbers, cursors and indices are one + * stored value by name. A save publishes what changed, all or nothing. */ + +bool pure_save_enabled(const PureApp *); +/* A text state of m: its stored value, or its initial constant when none. */ +PureResult pure_save_open_text(PureApp *, const PureModule *m, uint32_t state, PureDraft **out, PureVersion *saved); +/* A list state of m: its stored items, fields moved by name when the stored + * record differs from m's. */ +PureResult pure_save_open_list(PureApp *, const PureModule *m, uint32_t state, PureList **out, PureDraft **draft, + PureVersion *saved); +/* Open the stored numbers, cursors and indices, and put each stored one of the + * same name and kind that still fits into values, for the states of m that + * kept marks as new (-1); texts and lists are m's. */ +PureResult pure_save_open_values(PureApp *, const PureModule *m, double *values, const int *kept, + PureDraft *const *texts, PureList *const *lists); +/* Store a list whose record changed in its draft, header and items anew. */ +PureResult pure_save_rewrite_list(const PureModule *m, uint32_t state, PureList *list, PureDraft *draft); +/* The stored links of a starting app (module.h, PureApp); CONFLICT when an + * output with sends left is missing or has other fields. */ +PureResult pure_save_open_links(PureApp *, const PureModule *m); +/* Publish the documents and every state changed since the last save + * (values are the states' numbers); revisions are the documents'. */ +PureResult pure_save_commit(PureApp *, const double *values, void *const *documents, size_t document_count, + int64_t *revisions); + +/* A list with record's fields, taking from `from` each field of the same name + * and kind and giving the others their defaults (module.c). */ +PureResult pure_list_migrate(PureList **out, const PureList *from, const char *const *names, + const uint8_t *text, unsigned count, const PureModule *m, uint32_t record); +#endif diff --git a/core/module-store.c b/core/module-store.c new file mode 100644 index 0000000..7e759fd --- /dev/null +++ b/core/module-store.c @@ -0,0 +1,88 @@ +#include "module-store.h" + +static PureResult open_document(void *context, uint32_t collection, const char *key, + void **document, int64_t *revision) { + (void)collection; /* v2 modules have one collection: the store's key space */ + PureDraft *draft; + PureResult result = pure_draft_open(context, key, &draft); + if (result != PURE_OK) return result; + pure_draft_hold(draft, true); /* values read in the event stay readable */ + *document = draft; + *revision = pure_draft_revision(draft); + return PURE_OK; +} + +static PureVersion version(void *context, void *document) { + (void)context; + return pure_draft_version(document); +} + +static PureResult read(void *context, void *document, PureVersion at, size_t offset, size_t length, + bool (*visit)(void *, PureSpan), void *visit_context) { + (void)context; + return pure_draft_read_version(document, at, offset, length, visit, visit_context); +} + +static PureResult splice(void *context, void *document, size_t offset, size_t remove, PureSpan insert) { + (void)context; + return pure_draft_splice(document, offset, remove, insert); +} + +static PureResult commit(void *context, void *document, int64_t *revision) { + (void)context; + return pure_draft_commit(document, revision); +} + +static void close_document(void *context, void *document) { + (void)context; + pure_draft_close(document); +} + +static PureResult state(void *context, const char *name, PureDraft **out) { + return pure_state_open(context, name, out); +} + +enum { SAVED = PURE_APP_PENDING + 2 * PURE_MODULE_STATES + 1 }; + +/* The documents and the app's changed states in one tree commit. */ +static PureResult save(void *context, void *const *documents, size_t document_count, PureDraft *const *states, + size_t count, int64_t *revisions) { + (void)context; + PureDraft *drafts[SAVED] = {0}; + int64_t all[SAVED] = {0}; + if (document_count > PURE_APP_PENDING || count > SAVED - document_count) return PURE_LIMIT; + for (size_t i = 0; i < document_count; ++i) drafts[i] = documents[i]; + for (size_t i = 0; i < count; ++i) drafts[document_count + i] = states[i]; + PureResult result = pure_draft_commit_all(drafts, document_count + count, all); + for (size_t i = 0; i < document_count; ++i) revisions[i] = all[i]; + return result; +} + +/* Through a store handle: documents and states share its mapping. */ +static PureResult open_handle_document(void *context, uint32_t collection, const char *key, void **document, + int64_t *revision) { + (void)collection; + PureDraft *draft; + PureResult result = pure_store_draft(context, key, &draft); + if (result != PURE_OK) return result; + pure_draft_hold(draft, true); + *document = draft; + *revision = pure_draft_revision(draft); + return PURE_OK; +} + +static PureResult handle_state(void *context, const char *name, PureDraft **out) { + return pure_store_state(context, name, out); +} + +PureData pure_store_data(PureStore *store) { + return (PureData){.context = store, .open = open_handle_document, .version = version, .read = read, + .splice = splice, .commit = commit, .close = close_document, .state = handle_state, + .save = save}; +} + +PureData pure_tree_data(const char *path) { + return (PureData){.context = (void *)path, .open = open_document, .version = version, + .read = read, .splice = splice, .commit = commit, .close = close_document, + .state = state, .save = save}; +} diff --git a/core/module-store.h b/core/module-store.h new file mode 100644 index 0000000..d14ab3d --- /dev/null +++ b/core/module-store.h @@ -0,0 +1,11 @@ +#ifndef PURE_MODULE_STORE_H +#define PURE_MODULE_STORE_H +#include "module.h" +/* Keyed module collections and the app's states on the selected tree store: + * one event opens the message key as a held draft and commits it together + * with the states the event changed. The path outlives the app. */ +PureData pure_tree_data(const char *path); +/* The same through a store handle (state.h): opening a document or state + * checks only whether the store changed. The handle outlives the app. */ +PureData pure_store_data(PureStore *); +#endif diff --git a/core/module-text.c b/core/module-text.c new file mode 100644 index 0000000..68c03b0 --- /dev/null +++ b/core/module-text.c @@ -0,0 +1,189 @@ +#include "module-text.h" +#include +#include +#include +#include + +enum { CHUNK = 4096 }; + +/* state: pending continuation bytes (bits 0-1) and the allowed range of the + * next one (bits 2-4): 0 any, 1 A0-BF after E0, 2 80-9F after ED, 3 90-BF + * after F0, 4 80-8F after F4. */ +bool pure_utf8_step(unsigned *state, PureSpan span) { + unsigned remaining = *state & 3, range = *state >> 2; + for (size_t i = 0; i < span.size; ++i) { + /* Eight ASCII bytes at a time between characters. */ + uint64_t word; + while (!remaining && span.size - i >= 8 && (memcpy(&word, span.data + i, 8), !(word & UINT64_C(0x8080808080808080)))) + i += 8; + if (i == span.size) break; + unsigned char b = span.data[i]; + if (remaining) { + unsigned low = range == 1 ? 0xA0 : range == 3 ? 0x90 : 0x80; + unsigned high = range == 2 ? 0x9F : range == 4 ? 0x8F : 0xBF; + if (b < low || b > high) return false; + --remaining; + range = 0; + } else if (b < 0x80) { + continue; + } else if (b >= 0xC2 && b <= 0xDF) { + remaining = 1; + } else if (b >= 0xE0 && b <= 0xEF) { + remaining = 2; + range = b == 0xE0 ? 1 : b == 0xED ? 2 : 0; + } else if (b >= 0xF0 && b <= 0xF4) { + remaining = 3; + range = b == 0xF0 ? 3 : b == 0xF4 ? 4 : 0; + } else return false; + } + *state = remaining | range << 2; + return true; +} + +bool pure_utf8_complete(unsigned state) { return (state & 3) == 0; } + +bool pure_utf8_valid(PureSpan span) { + unsigned state = 0; + return pure_utf8_step(&state, span) && pure_utf8_complete(state); +} + +size_t pure_utf8_scalars(PureSpan span) { + size_t count = 0; + for (size_t i = 0; i < span.size; ++i) count += (span.data[i] & 0xC0) != 0x80; + return count; +} + +size_t pure_utf8_newlines(PureSpan span) { + size_t count = 0; + for (size_t i = 0; i < span.size; ++i) count += span.data[i] == '\n'; + return count; +} + +size_t pure_format_number(double value, char out[32]) { + if (value == 0) { memcpy(out, "0", 2); return 1; } + if (value == floor(value) && fabs(value) < 1e15) { + /* An integer's digits, without printf (the common case, many times cheaper). */ + char digits[16]; + uint64_t n = (uint64_t)fabs(value); + size_t count = 0, length = 0; + do { digits[count++] = (char)('0' + n % 10); n /= 10; } while (n); + if (value < 0) out[length++] = '-'; + while (count) out[length++] = digits[--count]; + out[length] = 0; + return length; + } + for (int precision = 1; precision <= 17; ++precision) { + snprintf(out, 32, "%.*g", precision, value); + if (strtod(out, NULL) == value) break; + } + char *exponent = strchr(out, 'e'); + if (exponent) { + char *digits = exponent + 2; /* after the sign */ + while (digits[0] == '0' && digits[1]) memmove(digits, digits + 1, strlen(digits)); + } + return strlen(out); +} + +typedef struct { unsigned char *out; } Copy; + +static bool copy(void *context, PureSpan span) { + Copy *c = context; + memcpy(c->out, span.data, span.size); + c->out += span.size; + return true; +} + +static void read_bytes(const PureDraft *draft, PureVersion version, size_t offset, size_t length, + unsigned char *out) { + Copy c = {out}; + if (length) pure_draft_read_version(draft, version, offset, length, copy, &c); +} + +static size_t forward(const PureDraft *draft, PureVersion version, size_t position) { + if (position >= version.bytes) return version.bytes; + unsigned char lead; + read_bytes(draft, version, position, 1, &lead); + size_t length = lead < 0x80 ? 1 : lead < 0xE0 ? 2 : lead < 0xF0 ? 3 : 4; + return position + length <= version.bytes ? position + length : version.bytes; +} + +static size_t backward(const PureDraft *draft, PureVersion version, size_t position) { + if (!position) return 0; + size_t take = position < 4 ? position : 4; + unsigned char bytes[4]; + read_bytes(draft, version, position - take, take, bytes); + size_t at = take - 1; + while (at && (bytes[at] & 0xC0) == 0x80) --at; + return position - take + at; +} + +/* Saturating: stops at the start or end, so work is bounded by the text size. */ +size_t pure_text_step(const PureDraft *draft, PureVersion version, size_t position, double count) { + for (; count > 0 && position < version.bytes; --count) position = forward(draft, version, position); + for (; count < 0 && position > 0; ++count) position = backward(draft, version, position); + return position; +} + +size_t pure_text_line_start(const PureDraft *draft, PureVersion version, size_t position) { + unsigned char chunk[CHUNK]; + while (position) { + size_t take = position < CHUNK ? position : CHUNK; + read_bytes(draft, version, position - take, take, chunk); + for (size_t i = take; i-- > 0;) + if (chunk[i] == '\n') return position - take + i + 1; + position -= take; + } + return 0; +} + +size_t pure_text_line_end(const PureDraft *draft, PureVersion version, size_t position) { + unsigned char chunk[CHUNK]; + while (position < version.bytes) { + size_t take = version.bytes - position < CHUNK ? version.bytes - position : CHUNK; + read_bytes(draft, version, position, take, chunk); + for (size_t i = 0; i < take; ++i) + if (chunk[i] == '\n') return position + i; + position += take; + } + return version.bytes; +} + +static size_t advance_within(const PureDraft *draft, PureVersion version, size_t position, + size_t columns, size_t limit) { + for (; columns && position < limit; --columns) position = forward(draft, version, position); + return position < limit ? position : limit; +} + +/* Moves keep the column; with no further line the cursor goes to the end or + * the start of the text. */ +size_t pure_text_line_step(const PureDraft *draft, PureVersion version, size_t position, double count) { + size_t start = pure_text_line_start(draft, version, position), column = 0; + for (size_t at = start; at < position; at = forward(draft, version, at)) ++column; + for (; count > 0; --count) { + size_t end = pure_text_line_end(draft, version, position); + if (end == version.bytes) return version.bytes; + size_t next = end + 1; + position = advance_within(draft, version, next, column, pure_text_line_end(draft, version, next)); + } + for (; count < 0; ++count) { + size_t here = pure_text_line_start(draft, version, position); + if (!here) return 0; + size_t previous = pure_text_line_start(draft, version, here - 1); + position = advance_within(draft, version, previous, column, here - 1); + } + return position; +} + +typedef struct { size_t count; unsigned unit; } Count; + +static bool count_span(void *context, PureSpan span) { + Count *c = context; + c->count += c->unit ? pure_utf8_newlines(span) : pure_utf8_scalars(span); + return true; +} + +size_t pure_text_count(const PureDraft *draft, PureVersion version, size_t position, unsigned unit) { + Count c = {0, unit}; + if (position) pure_draft_read_version(draft, version, 0, position, count_span, &c); + return c.count + (unit ? 1 : 0); +} diff --git a/core/module-text.h b/core/module-text.h new file mode 100644 index 0000000..4843928 --- /dev/null +++ b/core/module-text.h @@ -0,0 +1,34 @@ +#ifndef PURE_MODULE_TEXT_H +#define PURE_MODULE_TEXT_H +#include "flow.h" +#include "state.h" +#include +#include + +/* Text helpers of the module executor. Text is valid UTF-8; a character is a + * Unicode scalar value, so every position these return is a scalar boundary. */ + +/* Strict UTF-8: no overlongs, surrogates or values past U+10FFFF. state + * carries a sequence split across spans; start it at 0 and finish with + * pure_utf8_complete. */ +bool pure_utf8_step(unsigned *state, PureSpan span); +bool pure_utf8_complete(unsigned state); +bool pure_utf8_valid(PureSpan span); + +/* Scalars (bytes that are not 10xxxxxx) and newlines in a span. */ +size_t pure_utf8_scalars(PureSpan span); +size_t pure_utf8_newlines(PureSpan span); + +/* Shortest decimal that reads back to the same finite double: integers + * below 1e15 without a point or exponent, otherwise %g form with a trimmed + * exponent. -0 prints as 0. Returns the length written (at most 32). */ +size_t pure_format_number(double value, char out[32]); + +/* Positions in one version of a text draft (size is version.bytes). */ +size_t pure_text_step(const PureDraft *, PureVersion, size_t position, double count); +size_t pure_text_line_step(const PureDraft *, PureVersion, size_t position, double count); +size_t pure_text_line_start(const PureDraft *, PureVersion, size_t position); +size_t pure_text_line_end(const PureDraft *, PureVersion, size_t position); +/* Scalars (unit 0) or line number counting from 1 (unit 1) before position. */ +size_t pure_text_count(const PureDraft *, PureVersion, size_t position, unsigned unit); +#endif diff --git a/core/module.c b/core/module.c new file mode 100644 index 0000000..697e918 --- /dev/null +++ b/core/module.c @@ -0,0 +1,2122 @@ +#include "module.h" +#include "module-save.h" +#include "module-text.h" +#include +#include +#include +#include + +enum { STATE_BYTES = 16, COLLECTION_BYTES = 12, INPUT_BYTES = 16, FIELD_BYTES = 8, + CONSTANT_BYTES = 8, RECORD_BYTES = 8, RECORD_FIELD_BYTES = 16, VIEW_BYTES = 16, + HANDLER_BYTES = 12, OP_BYTES = 16 }; +/* T_FIELD and T_ITEM stand for the type of the message or record field + * named, T_KEY for the type of the named keyed list's key. */ +enum { T_NUM = 1, T_BYTES = 2, T_TEXT = 3, T_CURSOR = 4, T_FIELD = 5, T_INDEX = 6, T_ITEM = 7, T_KEY = 8 }; +enum { FROM_PAYLOAD, FROM_DOCUMENT, FROM_COPY, FROM_CONSTANT, FROM_TEXT, FROM_ARENA }; +/* Where an opcode may appear. */ +enum { IN_VIEW = 1, IN_HANDLER = 2, IN_MESSAGE = 4, IN_KEYED = 8, IN_TEXT = 16, IN_ROW = 32 }; +enum { READ = IN_VIEW | IN_HANDLER }; +/* What an argument names: nothing (it is 0), a state of one type (A_STATE + + * the type), a list and one of its record's fields, a message field, the + * collection, a constant, a unit, the column's row, a branch target, a stack + * depth, a fold, a relation or a keyed list. */ +enum { A_NONE, A_STATE, A_ITEM = A_STATE + PURE_STATE_INDEX + 1, A_FIELD, A_COLLECTION, A_CONSTANT, A_UNIT, + A_ROW, A_TARGET, A_DEPTH, A_FOLD, A_RELATION, A_KEYED, A_OUTPUT }; +/* A fold's block is verified as a column that leaves one number (count) or + * two (a value and whether to include it), a table's filter as a column that + * leaves one: these stand for a view type. */ +enum { FOLD_ONE = 100, FOLD_TWO, FILTER }; +/* The text or list an opcode works on: its cursor and index operands belong + * to it, and so does a cursor or index it pushes. O_ANY: no rule. O_OPERAND: + * it pushes a position in its operand's text or list. O_ARG: the argument. + * O_REF: the text or list of the argument's cursor or index state. O_LIST: + * the list of an item argument. */ +enum { O_ANY, O_OPERAND, O_ARG, O_REF, O_LIST }; +static const unsigned char magic[8] = {'P','U','R','E','M','O','D','5'}; + +/* Operand types popped (bottom to top), the type pushed, where allowed, what + * the argument names and the text or list worked on. */ +#define NUMS(pops) {pops, {T_NUM, T_NUM, T_NUM}, T_NUM, READ, A_NONE, O_ANY} +#define STATE(type) A_STATE + PURE_STATE_##type +static const struct { unsigned char pops, in[3], out, where, arg, of; } signature[PURE_OP_COUNT] = { + [PURE_OP_CONST] = NUMS(0), [PURE_OP_ADD] = NUMS(2), [PURE_OP_SUB] = NUMS(2), [PURE_OP_MUL] = NUMS(2), + [PURE_OP_DIV] = NUMS(2), [PURE_OP_NEG] = NUMS(1), [PURE_OP_LT] = NUMS(2), [PURE_OP_LE] = NUMS(2), + [PURE_OP_GT] = NUMS(2), [PURE_OP_GE] = NUMS(2), [PURE_OP_EQ] = NUMS(2), [PURE_OP_NE] = NUMS(2), + [PURE_OP_AND] = NUMS(2), [PURE_OP_OR] = NUMS(2), [PURE_OP_NOT] = NUMS(1), [PURE_OP_MIN] = NUMS(2), + [PURE_OP_MAX] = NUMS(2), [PURE_OP_CLAMP] = NUMS(3), + [PURE_OP_LOAD] = {0, {0}, T_NUM, READ, STATE(NUM), O_ANY}, + [PURE_OP_DT] = {0, {0}, T_NUM, IN_HANDLER, A_NONE, O_ANY}, + [PURE_OP_STORE] = {1, {T_NUM}, 0, IN_HANDLER, STATE(NUM), O_ANY}, + [PURE_OP_JZ] = {1, {T_NUM}, 0, READ, A_TARGET, O_ANY}, + [PURE_OP_JMP] = {0, {0}, 0, READ, A_TARGET, O_ANY}, + [PURE_OP_FIELD] = {0, {0}, T_FIELD, IN_MESSAGE, A_FIELD, O_ANY}, + [PURE_OP_DOC] = {0, {0}, T_BYTES, IN_KEYED, A_COLLECTION, O_ANY}, + [PURE_OP_DOC_SIZE] = {0, {0}, T_NUM, IN_KEYED, A_COLLECTION, O_ANY}, + [PURE_OP_LEN] = {1, {T_BYTES}, T_NUM, IN_HANDLER, A_NONE, O_ANY}, + [PURE_OP_REPLACE] = {3, {T_NUM, T_NUM, T_BYTES}, 0, IN_KEYED, A_COLLECTION, O_ANY}, + [PURE_OP_REPLY] = {1, {T_BYTES}, 0, IN_MESSAGE, A_NONE, O_ANY}, + [PURE_OP_REQUIRE] = {1, {T_NUM}, 0, IN_HANDLER, A_NONE, O_ANY}, + [PURE_OP_TEXT_CONST] = {0, {0}, T_TEXT, READ, A_CONSTANT, O_ANY}, + [PURE_OP_TEXT_LOAD] = {0, {0}, T_TEXT, READ, STATE(TEXT), O_ANY}, + [PURE_OP_TEXT_INPUT] = {0, {0}, T_TEXT, IN_TEXT, A_NONE, O_ANY}, + [PURE_OP_NUM_TEXT] = {1, {T_NUM}, T_TEXT, READ, A_NONE, O_ANY}, + [PURE_OP_CONCAT] = {2, {T_TEXT, T_TEXT}, T_TEXT, READ, A_NONE, O_ANY}, + [PURE_OP_CHARS] = {1, {T_TEXT}, T_NUM, READ, A_NONE, O_ANY}, + [PURE_OP_LINES] = {1, {T_TEXT}, T_NUM, READ, A_NONE, O_ANY}, + [PURE_OP_TEXT_BYTES] = {1, {T_TEXT}, T_NUM, READ, A_NONE, O_ANY}, + [PURE_OP_CUR_LOAD] = {0, {0}, T_CURSOR, READ, STATE(CURSOR), O_REF}, + [PURE_OP_CUR_START] = {0, {0}, T_CURSOR, READ, STATE(TEXT), O_ARG}, + [PURE_OP_CUR_END] = {0, {0}, T_CURSOR, READ, STATE(TEXT), O_ARG}, + [PURE_OP_CUR_MOVE] = {2, {T_CURSOR, T_NUM}, T_CURSOR, READ, A_UNIT, O_OPERAND}, + [PURE_OP_CUR_STORE] = {1, {T_CURSOR}, 0, IN_HANDLER, STATE(CURSOR), O_REF}, + [PURE_OP_TEXT_INSERT] = {2, {T_CURSOR, T_TEXT}, 0, IN_HANDLER, STATE(TEXT), O_ARG}, + [PURE_OP_TEXT_REMOVE] = {2, {T_CURSOR, T_CURSOR}, 0, IN_HANDLER, STATE(TEXT), O_ARG}, + [PURE_OP_TEXT_LINE] = {1, {T_CURSOR}, T_TEXT, READ, STATE(TEXT), O_ARG}, + [PURE_OP_TEXT_SET] = {1, {T_TEXT}, 0, IN_HANDLER, STATE(TEXT), O_ANY}, + [PURE_OP_TEXT_FROM] = {1, {T_BYTES}, T_TEXT, READ, A_NONE, O_ANY}, + [PURE_OP_BYTES_FROM] = {1, {T_TEXT}, T_BYTES, READ, A_NONE, O_ANY}, + [PURE_OP_CUR_COUNT] = {1, {T_CURSOR}, T_NUM, READ, A_UNIT, O_ANY}, + /* Record operations also take as many values as the list's record has fields. */ + [PURE_OP_LIST_COUNT] = {0, {0}, T_NUM, READ, STATE(LIST), O_ANY}, + [PURE_OP_LIST_APPEND] = {0, {0}, 0, IN_HANDLER, STATE(LIST), O_ANY}, + [PURE_OP_LIST_INSERT] = {1, {T_INDEX}, 0, IN_HANDLER, STATE(LIST), O_ARG}, + [PURE_OP_LIST_REMOVE] = {1, {T_INDEX}, 0, IN_HANDLER, STATE(LIST), O_ARG}, + [PURE_OP_ITEM_GET] = {1, {T_INDEX}, T_ITEM, READ, A_ITEM, O_LIST}, + [PURE_OP_ITEM_SET] = {1, {T_INDEX}, 0, IN_HANDLER, A_ITEM, O_LIST}, + [PURE_OP_INDEX_LOAD] = {0, {0}, T_INDEX, READ, STATE(INDEX), O_REF}, + [PURE_OP_INDEX_STORE] = {1, {T_INDEX}, 0, IN_HANDLER, STATE(INDEX), O_REF}, + [PURE_OP_INDEX_MOVE] = {2, {T_INDEX, T_NUM}, T_INDEX, READ, A_NONE, O_OPERAND}, + [PURE_OP_INDEX_FIRST] = {0, {0}, T_INDEX, READ, STATE(LIST), O_ARG}, + [PURE_OP_INDEX_LAST] = {0, {0}, T_INDEX, READ, STATE(LIST), O_ARG}, + [PURE_OP_INDEX_NUM] = {1, {T_INDEX}, T_NUM, READ, A_NONE, O_ANY}, + [PURE_OP_ROW] = {0, {0}, T_INDEX, IN_ROW, A_ROW, O_ARG}, + /* Stack copies: their types come from the values they move. */ + [PURE_OP_PICK] = {0, {0}, 0, READ, A_DEPTH, O_ANY}, + [PURE_OP_SLIDE] = {0, {0}, 0, READ, A_DEPTH, O_ANY}, + [PURE_OP_FOLD] = {0, {0}, T_NUM, READ, A_FOLD, O_ANY}, + [PURE_OP_TEXT_COMPARE] = {2, {T_TEXT, T_TEXT}, T_NUM, READ, A_RELATION, O_ANY}, + [PURE_OP_LIST_FIND] = {1, {T_KEY}, T_INDEX, READ, A_KEYED, O_ARG}, + [PURE_OP_LIST_HAS] = {1, {T_KEY}, T_NUM, READ, A_KEYED, O_ANY}, + [PURE_OP_NOW] = {0, {0}, T_NUM, IN_HANDLER, A_NONE, O_ANY}, + [PURE_OP_SEND] = {0, {0}, 0, IN_HANDLER, A_OUTPUT, O_ANY}, + [PURE_OP_FLOOR] = NUMS(1), [PURE_OP_MOD] = NUMS(2), +}; +#undef NUMS +#undef STATE + +static uint32_t read16(const unsigned char *p) { return (uint32_t)p[0] | (uint32_t)p[1] << 8; } + +static uint32_t read32(const unsigned char *p) { + return (uint32_t)p[0] | (uint32_t)p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; +} + +static void write32(unsigned char *p, uint32_t value) { + for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(value >> (8 * i)); +} + +static double read_double(const unsigned char *p) { + uint64_t bits = read32(p) | (uint64_t)read32(p + 4) << 32; + double value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +PURE_COLD static bool key_name(const char *key) { + static const char *const named[] = {"Space", "Enter", "ArrowLeft", "ArrowRight", "ArrowUp", "ArrowDown", + "Backspace", "Delete", "Home", "End", "Tab", "Escape", "PageUp", "PageDown"}; + for (size_t i = 0; i < sizeof(named) / sizeof(*named); ++i) + if (!strcmp(key, named[i])) return true; + if (!strncmp(key, "Key", 3)) return key[3] >= 'A' && key[3] <= 'Z' && !key[4]; + if (!strncmp(key, "Digit", 5)) return key[5] >= '0' && key[5] <= '9' && !key[6]; + return false; +} + +/* A document key: 1..64 bytes of [A-Za-z0-9_-], as every Pure store accepts. */ +static bool document_key(const char *key) { + size_t length = key ? strlen(key) : 0; + if (!length || length > 64) return false; + for (size_t i = 0; i < length; ++i) { + char c = key[i]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '-' || c == '_')) return false; + } + return true; +} + +/* Strings are [u8 length][bytes]; a name is 1..31 bytes: an ASCII identifier, + * a path of identifiers joined by dots (a message field inside a group, as + * user.name), or any printable ASCII (a key's name). */ +enum { NAME_IDENTIFIER, NAME_PATH, NAME_PRINTABLE }; +PURE_COLD static bool name_at(const unsigned char *strings, uint32_t size, uint32_t offset, + int kind, char out[PURE_MODULE_NAME]) { + if (offset >= size) return false; + uint32_t length = strings[offset]; + if (!length || length >= PURE_MODULE_NAME || length > size - offset - 1) return false; + for (uint32_t i = 0, start = 0; i < length; ++i) { + unsigned char c = strings[offset + 1 + i]; + bool letter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; + if (kind == NAME_PATH && c == '.' && i > start && i + 1 < length) { start = i + 1; continue; } + if (!letter && !(i > start && c >= '0' && c <= '9') && (kind != NAME_PRINTABLE || c < 32 || c > 126)) + return false; + } + memcpy(out, strings + offset + 1, length); + out[length] = 0; + return true; +} + +/* The input whose handler this is, or -1 for frame. */ +static int event_input(uint32_t event) { return event ? (int)((event - 1) / 2) : -1; } + +/* The state an argument names must have this type. */ +static bool state_of(const PureModule *m, uint32_t state, uint8_t type) { + return state < m->states && m->state_type[state] == type; +} + +/* The list an item operation names: its low 16 bits (the field is above). */ +static uint32_t list_of(uint32_t arg) { return arg & 0xFFFF; } +static uint32_t item_field(uint32_t arg) { return arg >> 16; } + +static unsigned char field_type(const PureModule *m, uint32_t list, uint32_t field) { + return m->record_field_text[m->record[m->state_ref[list]].first + field] ? T_TEXT : T_NUM; +} + +PURE_COLD static bool names_arg(const PureModule *m, unsigned kind, uint32_t arg, const PureInputShape *shape, int row) { + switch (kind) { + case A_ITEM: + return state_of(m, list_of(arg), PURE_STATE_LIST) && item_field(arg) < m->record[m->state_ref[list_of(arg)]].count; + case A_FIELD: return arg < shape->field_count; + case A_COLLECTION: return arg < m->collections; + case A_CONSTANT: return arg < m->constants; + case A_UNIT: return arg <= PURE_UNIT_LINE; + case A_ROW: return (int)arg == row; + case A_FOLD: return (arg & 0xFF) <= PURE_FOLD_FIRST && state_of(m, arg >> 8 & 0xFF, PURE_STATE_LIST) && arg >> 16; + case A_RELATION: return arg <= PURE_NOT_EQUAL; + case A_KEYED: return state_of(m, arg, PURE_STATE_LIST) && m->state_key[arg]; + case A_OUTPUT: return arg < m->inputs && m->input[arg].kind == PURE_KIND_OUTPUT; + case A_NONE: case A_TARGET: case A_DEPTH: return true; + default: return state_of(m, arg, (uint8_t)(kind - A_STATE)); + } +} + +/* The stack a branch brings to its target, while the target is ahead. */ +typedef struct { uint32_t target; int height; unsigned char types[PURE_MODULE_STACK], owners[PURE_MODULE_STACK]; } Join; + +static bool same_stack(const Join *join, int height, const unsigned char *types, const unsigned char *owners) { + return join->height == height && !memcmp(join->types, types, (size_t)height) && + !memcmp(join->owners, owners, (size_t)height); +} + +/* Types and heights are checked per instruction; a cursor's type records + * which text it belongs to and an index's type which list (other values + * belong to nothing). Branches go forward within the block. Every path into + * a join brings the same stack (types and owners), so a value may cross a + * join, as in a conditional expression; at most 8 joins with a non-empty + * stack are ahead at once. Handlers end empty, views with one value of their + * type. A column (row is its list) may read the row it is evaluated for. + * A fold's block is verified as a column of its list (reads only, no fold + * in it); the block around treats it as one instruction and may not branch + * into it. Returns why the block is refused, with at moved to the + * instruction when one is at fault, or NULL. */ +PURE_COLD static const char *verify_block(const PureModule *, PureBlock, int view, int row, uint32_t event, uint32_t *at); + +/* target marks, per instruction of the block, a branch into it with an empty stack. */ +PURE_COLD static const char *verify_code(const PureModule *m, PureBlock block, int view, int row, uint32_t event, + uint32_t *at, unsigned char *target) { +#define REFUSE(why) do { *at = pc; return why; } while (0) + if (view >= 0 && !block.length) return "empty view"; + int input = view >= 0 ? -1 : event_input(event); + const PureInputShape *shape = input >= 0 ? &m->input[input] : NULL; + unsigned where = view >= 0 ? IN_VIEW | (row >= 0 ? IN_ROW : 0) : IN_HANDLER; + if (shape && shape->kind == PURE_KIND_MESSAGE) where |= IN_MESSAGE | (shape->keyed ? IN_KEYED : 0); + if (shape && shape->kind == PURE_KIND_TEXT) where |= IN_TEXT; + unsigned char types[PURE_MODULE_STACK], owners[PURE_MODULE_STACK]; + Join joins[PURE_MODULE_JOINS]; + unsigned open = 0; + int height = 0; + bool reachable = true; + for (uint32_t i = 0; i <= block.length; ++i) { + uint32_t pc = block.start + i; + /* Branches here brought an empty stack (target) or the stack of a join. */ + unsigned join = open; + for (unsigned k = 0; k < open; ++k) if (joins[k].target == pc) join = k; + if (join < open) { + if (target[i] || (reachable && !same_stack(&joins[join], height, types, owners))) + REFUSE("join with different stacks"); + height = joins[join].height; + memcpy(types, joins[join].types, (size_t)height); + memcpy(owners, joins[join].owners, (size_t)height); + joins[join] = joins[--open]; + reachable = true; + } else if (target[i]) { + if (reachable && height) REFUSE("join with different stacks"); + height = 0; + reachable = true; + } + if (i == block.length) break; + if (!reachable) REFUSE("unreachable instruction"); + const PureOp *op = &m->code[pc]; + if (op->op >= PURE_OP_COUNT) REFUSE("unknown opcode"); + unsigned allowed = signature[op->op].where, kind = signature[op->op].arg, of = signature[op->op].of; + if (!(allowed & where)) { + if (allowed & IN_ROW) REFUSE("row outside a column"); + if (view == FILTER) REFUSE("filter with an effect"); + if (view >= FOLD_ONE) REFUSE("fold with an effect"); + if (view >= 0) REFUSE("view with an effect"); + if (allowed & IN_KEYED) REFUSE("document operation outside a keyed message handler"); + if (allowed & IN_TEXT) REFUSE("text input outside a text handler"); + REFUSE("message operation outside a message handler"); + } + uint32_t arg = op->arg; + if ((kind == A_NONE && arg) || (op->op == PURE_OP_CONST ? !isfinite(op->value) : op->value != 0)) + REFUSE("non-canonical operand"); + if (!names_arg(m, kind, arg, shape, row)) REFUSE("operand names no such state, field, constant or unit"); + /* A keyed list places its items by key, and an item's key stays. */ + if (op->op == PURE_OP_LIST_INSERT && m->state_key[arg]) REFUSE("insert by position into a keyed list"); + if (op->op == PURE_OP_ITEM_SET && m->state_key[list_of(arg)] == item_field(arg) + 1) + REFUSE("change of a keyed list's key"); + /* PICK copies the value arg below the top; SLIDE drops arg values under the top. */ + if (kind == A_DEPTH) { + bool pick = op->op == PURE_OP_PICK; + if (pick ? arg >= (uint32_t)height : !arg || arg >= (uint32_t)height) REFUSE("stack underflow"); + if (pick && height == PURE_MODULE_STACK) REFUSE("stack overflow"); + int from = height - 1 - (int)arg, to = pick ? height++ : from; + types[to] = pick ? types[from] : types[height - 1]; + owners[to] = pick ? owners[from] : owners[height - 1]; + if (!pick) height = from + 1; + continue; + } + uint32_t skip = 0; + if (kind == A_FOLD) { + uint32_t inner = pc; + skip = arg >> 16; + if (view >= FOLD_ONE) REFUSE(view == FILTER ? "fold inside a filter" : "fold inside a fold"); + if (skip > block.length - i - 1) REFUSE("fold block outside its block"); + unsigned how = arg & 0xFF; + const char *why = verify_block(m, (PureBlock){pc + 1, skip}, + how == PURE_FOLD_COUNT || how == PURE_FOLD_FIRST ? FOLD_ONE : FOLD_TWO, + (int)(arg >> 8 & 0xFF), 0, &inner); + if (why) { *at = inner; return why; } + for (uint32_t k = i + 1; k <= i + skip; ++k) { + if (target[k]) REFUSE("branch into a fold"); + for (unsigned j = 0; j < open; ++j) if (joins[j].target == block.start + k) REFUSE("branch into a fold"); + } + } + /* Operand types, bottom to top; record operations take their fields. */ + unsigned char need[PURE_MODULE_FIELDS + 2]; + unsigned pops = signature[op->op].pops; + memcpy(need, signature[op->op].in, sizeof(signature[op->op].in)); + if (op->op == PURE_OP_SEND) { + const PureInputShape *output = &m->input[arg]; + for (unsigned f = 0; f < output->field_count; ++f) { + uint8_t type = m->field[output->field_start + f].type; + need[f] = type == PURE_FIELD_TEXT ? T_TEXT : type == PURE_FIELD_BYTES ? T_BYTES : T_NUM; + } + pops = output->field_count; + } else if (op->op == PURE_OP_LIST_APPEND || op->op == PURE_OP_LIST_INSERT) { + const PureRecord *record = &m->record[m->state_ref[arg]]; + for (unsigned f = 0; f < record->count; ++f) need[pops + f] = field_type(m, arg, f); + pops += record->count; + } else if (op->op == PURE_OP_ITEM_SET) { + need[pops++] = field_type(m, list_of(arg), item_field(arg)); + } else if (need[0] == T_KEY) { + need[0] = field_type(m, arg, m->state_key[arg] - 1u); + } + if (height < (int)pops) REFUSE("stack underflow"); + height -= (int)pops; + for (unsigned k = 0; k < pops; ++k) + if (types[height + (int)k] != need[k]) REFUSE("operand type"); + unsigned owner = of == O_ARG ? arg : of == O_REF ? m->state_ref[arg] : of == O_LIST ? list_of(arg) : 0; + for (unsigned k = 0; of >= O_ARG && k < pops; ++k) { + unsigned char type = types[height + (int)k]; + if ((type == T_CURSOR || type == T_INDEX) && owners[height + (int)k] != owner) + REFUSE(type == T_CURSOR ? "cursor of another text" : "index of another list"); + } + unsigned char out = signature[op->op].out; + if (kind == A_FOLD && (arg & 0xFF) == PURE_FOLD_FIRST) { /* an index of its list */ + out = T_INDEX; + of = O_ARG; + owner = arg >> 8 & 0xFF; + } + if (out == T_FIELD) { + uint8_t type = m->field[shape->field_start + arg].type; + out = type == PURE_FIELD_TEXT ? T_TEXT : type == PURE_FIELD_BYTES ? T_BYTES : T_NUM; + } + if (out == T_ITEM) out = field_type(m, list_of(arg), item_field(arg)); + if (out) { + if (height == PURE_MODULE_STACK) REFUSE("stack overflow"); + owners[height] = (unsigned char)(out != T_CURSOR && out != T_INDEX ? 0 : of == O_OPERAND ? owners[height] : owner); + types[height++] = out; + } + if (kind == A_TARGET) { + if (arg <= pc || arg > block.start + block.length) REFUSE("branch not forward within its block"); + join = open; + for (unsigned k = 0; k < open; ++k) if (joins[k].target == arg) join = k; + if (!height) { + if (join < open) REFUSE("join with different stacks"); + target[arg - block.start] = 1; + } else if (join < open) { + if (!same_stack(&joins[join], height, types, owners)) REFUSE("join with different stacks"); + } else { + if (target[arg - block.start]) REFUSE("join with different stacks"); + if (open == PURE_MODULE_JOINS) REFUSE("more than 8 joins ahead"); + joins[open] = (Join){.target = arg, .height = height}; + memcpy(joins[open].types, types, (size_t)height); + memcpy(joins[open++].owners, owners, (size_t)height); + } + if (op->op == PURE_OP_JMP) reachable = false; + } + i += skip; + } + if (!reachable) return "block end is unreachable"; + if (view == FILTER) return height != 1 || types[0] != T_NUM ? "filter must leave one number" : NULL; + if (view >= FOLD_ONE) { + int want = view - FOLD_ONE + 1; + return height != want || types[0] != T_NUM || types[want - 1] != T_NUM ? "fold block must leave its numbers" : NULL; + } + unsigned char result = view == PURE_VIEW_TEXT ? T_TEXT : T_NUM; + if (view >= 0 ? height != 1 || types[0] != result : height != 0) + return view >= 0 ? "view must leave one value of its type" : "handler must leave no value"; + return NULL; +#undef REFUSE +} + +PURE_COLD static const char *verify_block(const PureModule *m, PureBlock block, int view, int row, uint32_t event, + uint32_t *at) { + if (block.start > m->code_count || block.length > m->code_count - block.start) return "code range outside module"; + unsigned char *target = calloc((size_t)block.length + 1, 1); + if (!target) return "out of memory"; + const char *why = verify_code(m, block, view, row, event, at, target); + free(target); + return why; +} + +PURE_COLD PureResult pure_module_read(const char *path, PureModule *out, char *error, size_t error_size) { + FILE *file = fopen(path, "rb"); + unsigned char *bytes = NULL; + size_t size = 0, capacity = 0; + bool ok = file != NULL; + /* The largest module is some 2 MiB: code, strings and constants at their limits. */ + while (ok && !feof(file) && capacity < (8u << 20)) { + capacity = capacity ? capacity * 2 : 1 << 16; + unsigned char *grown = realloc(bytes, capacity); + if (!grown) { ok = false; break; } + bytes = grown; + size += fread(bytes + size, 1, capacity - size, file); + ok = !ferror(file); + } + ok = ok && feof(file); + if (file) fclose(file); + PureResult result = PURE_NOT_FOUND; + if (!ok) { + if (error_size) snprintf(error, error_size, "cannot read %s", path); + pure_module_release(out); + } else result = pure_module_load(bytes, size, out, error, error_size); + free(bytes); + return result; +} + +void pure_module_release(PureModule *m) { + free(m->storage); + m->storage = NULL; + m->code = NULL; + m->constant_offset = m->constant_length = NULL; + m->constant_data = NULL; +} + +PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule *out, + char *error, size_t error_size) { + const unsigned char *p = bytes; + const char *why; + uint32_t bad = 0; + PureResult result = PURE_INVALID; +#define REFUSE(message, index) do { why = message; bad = (uint32_t)(index); goto refused; } while (0) + if (error_size) error[0] = 0; + pure_module_release(out); + memset(out, 0, sizeof(*out)); + if (!p || size < PURE_MODULE_HEADER || memcmp(p, magic, sizeof(magic))) REFUSE("not a Pure module v5", 0); + out->states = (uint16_t)read16(p + 8); + out->inputs = (uint16_t)read16(p + 10); + out->views = (uint16_t)read16(p + 12); + out->handlers = (uint16_t)read16(p + 14); + out->code_count = read32(p + 16); + uint32_t strings = read32(p + 20); + out->fields = (uint16_t)read16(p + 24); + out->collections = (uint16_t)read16(p + 26); + out->constants = (uint16_t)read16(p + 28); + out->records = (uint16_t)read16(p + 30); + uint32_t constant_bytes = read32(p + 32); + out->record_fields = (uint16_t)read16(p + 36); + if (read16(p + 38)) REFUSE("reserved header bytes", 38); + if (out->states > PURE_MODULE_STATES || out->inputs > PURE_MODULE_INPUTS || + out->views > PURE_MODULE_VIEWS || out->handlers > PURE_MODULE_HANDLERS || + out->code_count > PURE_MODULE_CODE || strings > PURE_MODULE_STRINGS || + out->fields > PURE_MODULE_FIELDS || out->collections > PURE_MODULE_COLLECTIONS || + out->constants > PURE_MODULE_CONSTANTS || constant_bytes > PURE_MODULE_CONSTANT_BYTES || + out->records > PURE_MODULE_RECORDS || out->record_fields > PURE_MODULE_RECORD_FIELDS) { + result = PURE_LIMIT; + REFUSE("module exceeds a limit", 8); + } + size_t expected = PURE_MODULE_HEADER + (size_t)out->states * STATE_BYTES + + (size_t)out->collections * COLLECTION_BYTES + (size_t)out->inputs * INPUT_BYTES + + (size_t)out->fields * FIELD_BYTES + (size_t)out->constants * CONSTANT_BYTES + + (size_t)out->records * RECORD_BYTES + (size_t)out->record_fields * RECORD_FIELD_BYTES + + (size_t)out->views * VIEW_BYTES + (size_t)out->handlers * HANDLER_BYTES + + (size_t)out->code_count * OP_BYTES + strings + constant_bytes; + if (size != expected) REFUSE("module size", expected); + /* The code, the constants' offsets and lengths, then their bytes, in one block. */ + size_t storage = (size_t)out->code_count * sizeof(PureOp) + (size_t)out->constants * 2 * sizeof(uint32_t) + + constant_bytes; + if (!(out->storage = malloc(storage ? storage : 1))) { + result = PURE_ALLOCATION; + REFUSE("out of memory", 0); + } + out->code = out->storage; + out->constant_offset = (uint32_t *)(out->code + out->code_count); + out->constant_length = out->constant_offset + out->constants; + out->constant_data = (unsigned char *)(out->constant_length + out->constants); + const unsigned char *at = p + PURE_MODULE_HEADER; + const unsigned char *text = p + size - constant_bytes - strings; + if (constant_bytes) memcpy(out->constant_data, p + size - constant_bytes, constant_bytes); + for (uint32_t i = 0; i < out->states; ++i, at += STATE_BYTES) { + out->state_type[i] = (uint8_t)read32(at + 4); + if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->state_names[i])) REFUSE("state name", i); + if (read32(at + 4) > PURE_STATE_INDEX) REFUSE("state type", i); + if (out->state_type[i] == PURE_STATE_NUM) { + out->initial[i] = read_double(at + 8); + if (!isfinite(out->initial[i])) REFUSE("non-finite initial value", i); + } else { + /* Flags: a mark (cursor), or a keyed list's key field + 1. */ + uint32_t flags = read32(at + 12); + bool cursor = out->state_type[i] == PURE_STATE_CURSOR, list = out->state_type[i] == PURE_STATE_LIST; + if ((flags && !(cursor && flags == PURE_CURSOR_STAYS) && !(list && flags <= PURE_RECORD_FIELDS)) || + read32(at + 8) > UINT16_MAX) REFUSE("state reference", i); + out->state_ref[i] = (uint16_t)read32(at + 8); + out->state_flags[i] = cursor ? (uint8_t)flags : 0; + out->state_key[i] = list ? (uint8_t)flags : 0; + } + } + for (uint32_t i = 0; i < out->collections; ++i, at += COLLECTION_BYTES) { + if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->collection_names[i])) REFUSE("collection name", i); + if (read32(at + 4) != 1 || read32(at + 8) != 1) REFUSE("collection must hold bytes by key", i); + } + uint32_t next_field = 0; + const unsigned char *fields = at + (size_t)out->inputs * INPUT_BYTES; + for (uint32_t i = 0; i < out->inputs; ++i, at += INPUT_BYTES) { + PureInputShape *shape = &out->input[i]; + shape->kind = at[4]; + shape->keyed = at[5]; + shape->field_start = (uint16_t)read16(at + 12); + shape->field_count = (uint16_t)read16(at + 14); + if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->input_names[i])) REFUSE("input name", i); + if (read16(at + 6) || at[5] > 1) REFUSE("input flags", i); + if (shape->kind == PURE_KIND_KEY) { + if (shape->keyed || shape->field_start || shape->field_count || + !name_at(text, strings, read32(at + 8), NAME_PRINTABLE, out->input_keys[i]) || + !key_name(out->input_keys[i])) REFUSE("input key", i); + for (uint32_t k = 0; k < i; ++k) + if (out->input[k].kind == PURE_KIND_KEY && !strcmp(out->input_keys[k], out->input_keys[i])) + REFUSE("duplicate key", i); + continue; + } + if (shape->kind == PURE_KIND_TEXT) { + if (shape->keyed || shape->field_start || shape->field_count || read32(at + 8)) + REFUSE("text input shape", i); + continue; + } + if (shape->kind == PURE_KIND_TIMER) { + shape->period = read32(at + 8); + if (shape->keyed || shape->field_start || shape->field_count || !shape->period || + shape->period > PURE_TIMER_LONGEST) REFUSE("timer input", i); + continue; + } + if ((shape->kind != PURE_KIND_MESSAGE && shape->kind != PURE_KIND_OUTPUT) || read32(at + 8) || + (shape->kind == PURE_KIND_OUTPUT && shape->keyed) || shape->field_start != next_field || + shape->field_count > out->fields - next_field) REFUSE("message shape", i); + /* The prefix holds the fixed fields and the variable fields' lengths + * in order, but the last variable field's: it takes the rest. */ + uint32_t last = UINT32_MAX; + for (uint32_t f = 0; f < shape->field_count; ++f) { + uint32_t type = read32(fields + (size_t)(next_field + f) * FIELD_BYTES + 4); + if (type == PURE_FIELD_BYTES || type == PURE_FIELD_TEXT) last = f; + } + for (uint32_t f = 0; f < shape->field_count; ++f) { + const unsigned char *entry = fields + (size_t)(next_field + f) * FIELD_BYTES; + PureField *field = &out->field[next_field + f]; + field->type = (uint8_t)read32(entry + 4); + if (!name_at(text, strings, read32(entry), NAME_PATH, field->name) || field->type < PURE_FIELD_U32 || + read32(entry + 4) > PURE_FIELD_TEXT) + REFUSE("message field", next_field + f); + /* A name is a field or a group, never both (user and user.name). */ + for (uint32_t k = 0; k < f; ++k) { + const char *a = out->field[next_field + k].name, *b = field->name; + size_t la = strlen(a), lb = strlen(b); + if (!strcmp(a, b)) REFUSE("duplicate field", next_field + f); + if ((la < lb && !memcmp(a, b, la) && b[la] == '.') || (lb < la && !memcmp(a, b, lb) && a[lb] == '.')) + REFUSE("field and group", next_field + f); + } + field->offset = f == last ? PURE_FIELD_REST : shape->prefix; + if (f != last) shape->prefix += field->type == PURE_FIELD_NUMBER ? 8 : 4; + } + next_field += shape->field_count; + } + if (next_field != out->fields) REFUSE("unused message fields", next_field); + at = fields + (size_t)out->fields * FIELD_BYTES; + for (uint32_t i = 0; i < out->constants; ++i, at += CONSTANT_BYTES) { + out->constant_offset[i] = read32(at); + out->constant_length[i] = read32(at + 4); + if (out->constant_offset[i] > constant_bytes || out->constant_length[i] > constant_bytes - out->constant_offset[i] || + !pure_utf8_valid((PureSpan){out->constant_data + out->constant_offset[i], out->constant_length[i]})) + REFUSE("text constant", i); + } + /* Records: fields in order, numbered from first; a text default is a constant. */ + const unsigned char *fields_at = at + (size_t)out->records * RECORD_BYTES; + uint32_t next_record_field = 0; + for (uint32_t i = 0; i < out->records; ++i, at += RECORD_BYTES) { + PureRecord *record = &out->record[i]; + record->first = (uint16_t)read16(at + 4); + record->count = (uint16_t)read16(at + 6); + if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->record_names[i]) || record->first != next_record_field || + !record->count || record->count > PURE_RECORD_FIELDS || record->count > out->record_fields - next_record_field) + REFUSE("record", i); + for (uint32_t f = record->first; f < record->first + record->count; ++f) { + const unsigned char *entry = fields_at + (size_t)f * RECORD_FIELD_BYTES; + uint32_t type = read32(entry + 4); + out->record_field_text[f] = type == 1; + if (!name_at(text, strings, read32(entry), NAME_IDENTIFIER, out->record_field_names[f]) || type > 1) + REFUSE("record field", f); + for (uint32_t k = record->first; k < f; ++k) + if (!strcmp(out->record_field_names[k], out->record_field_names[f])) REFUSE("duplicate record field", f); + out->record_field_default[f] = type ? read32(entry + 8) : read_double(entry + 8); + if (type ? read32(entry + 12) || read32(entry + 8) >= out->constants : !isfinite(out->record_field_default[f])) + REFUSE("record field default", f); + } + next_record_field += record->count; + } + if (next_record_field != out->record_fields) REFUSE("unused record fields", next_record_field); + at = fields_at + (size_t)out->record_fields * RECORD_FIELD_BYTES; + for (uint32_t i = 0; i < out->states; ++i) + if ((out->state_type[i] == PURE_STATE_TEXT && out->state_ref[i] >= out->constants) || + (out->state_type[i] == PURE_STATE_CURSOR && !state_of(out, out->state_ref[i], PURE_STATE_TEXT)) || + (out->state_type[i] == PURE_STATE_LIST && + (out->state_ref[i] >= out->records || out->state_key[i] > out->record[out->state_ref[i]].count)) || + (out->state_type[i] == PURE_STATE_INDEX && !state_of(out, out->state_ref[i], PURE_STATE_LIST))) + REFUSE("state reference", i); + /* A table names its list in place of code; its columns follow it. */ + for (uint32_t i = 0; i < out->views; ++i, at += VIEW_BYTES) { + if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->view_names[i])) REFUSE("view name", i); + out->view_type[i] = at[4]; + out->view_table[i] = (uint16_t)read16(at + 6); + out->view_code[i] = (PureBlock){read32(at + 8), read32(at + 12)}; + uint32_t table = out->view_table[i]; + if (at[4] > PURE_VIEW_TABLE || (at[5] && at[4] != PURE_VIEW_TABLE) || + (table && (table > i || out->view_type[table - 1] != PURE_VIEW_TABLE || at[4] == PURE_VIEW_TABLE))) + REFUSE("view type", i); + /* A table names its list in start, or, filtered, in the byte after its + * type (list + 1) with its filter's block in start and length. */ + if (at[4] == PURE_VIEW_TABLE) { + uint32_t list = at[5] ? at[5] - 1u : out->view_code[i].start; + if (!state_of(out, list, PURE_STATE_LIST) || !at[5] != !out->view_code[i].length) REFUSE("table list", i); + out->view_list[i] = (uint16_t)list; + } else if (table) out->view_list[i] = out->view_list[table - 1]; + } + for (uint32_t i = 0; i < out->handlers; ++i, at += HANDLER_BYTES) { + uint32_t event = read32(at); + out->handler[i] = (PureHandler){event, {read32(at + 4), read32(at + 8)}}; + int input = event_input(event); + if (event > 2u * out->inputs || (input >= 0 && out->input[input].kind != PURE_KIND_KEY && !(event & 1)) || + (input >= 0 && out->input[input].kind == PURE_KIND_OUTPUT)) + REFUSE("handler event", i); + } + for (uint32_t i = 0; i < out->code_count; ++i, at += OP_BYTES) { + if (at[1] || at[2] || at[3]) REFUSE("instruction padding", i); + out->code[i] = (PureOp){.op = at[0], .arg = read32(at + 4), .value = read_double(at + 8)}; + } + /* States, records, collections, inputs and views share one namespace. */ + const char *names[PURE_MODULE_STATES + PURE_MODULE_RECORDS + PURE_MODULE_COLLECTIONS + + PURE_MODULE_INPUTS + PURE_MODULE_VIEWS]; + size_t count = 0; + for (uint32_t i = 0; i < out->states; ++i) names[count++] = out->state_names[i]; + for (uint32_t i = 0; i < out->records; ++i) names[count++] = out->record_names[i]; + for (uint32_t i = 0; i < out->collections; ++i) names[count++] = out->collection_names[i]; + for (uint32_t i = 0; i < out->inputs; ++i) names[count++] = out->input_names[i]; + for (uint32_t i = 0; i < out->views; ++i) if (!out->view_table[i]) names[count++] = out->view_names[i]; + for (size_t i = 0; i < count; ++i) + for (size_t k = 0; k < i; ++k) + if (!strcmp(names[i], names[k])) REFUSE("duplicate name", i); + /* Column names are unique within their table. */ + for (uint32_t i = 0; i < out->views; ++i) + for (uint32_t k = 0; k < i; ++k) + if (out->view_table[i] && out->view_table[k] == out->view_table[i] && + !strcmp(out->view_names[i], out->view_names[k])) REFUSE("duplicate column", i); + for (uint32_t i = 0; i < out->views; ++i) { + bad = i; + bool table = out->view_type[i] == PURE_VIEW_TABLE; + if ((!table || out->view_code[i].length) && + (why = verify_block(out, out->view_code[i], table ? FILTER : out->view_type[i], + table || out->view_table[i] ? out->view_list[i] : -1, 0, &bad))) goto refused; + } + for (uint32_t i = 0; i < out->handlers; ++i) { + bad = i; + if ((why = verify_block(out, out->handler[i].code, -1, -1, out->handler[i].event, &bad))) goto refused; + } + return PURE_OK; +refused: + if (error_size) snprintf(error, error_size, "%s (%u)", why, bad); + pure_module_release(out); + return result; +#undef REFUSE +} + +static int find(const char (*names)[PURE_MODULE_NAME], unsigned count, const char *name) { + for (unsigned i = 0; i < count; ++i) + if (!strcmp(names[i], name)) return (int)i; + return -1; +} + +int pure_module_state(const PureModule *m, const char *name) { return find(m->state_names, m->states, name); } +int pure_module_input(const PureModule *m, const char *name) { return find(m->input_names, m->inputs, name); } +int pure_module_view(const PureModule *m, const char *name) { + for (uint32_t i = 0; i < m->views; ++i) + if (!m->view_table[i] && !strcmp(m->view_names[i], name)) return (int)i; + return -1; +} + +int pure_module_position(const PureModule *m, uint32_t view) { + if (view >= m->views || m->view_type[view] != PURE_VIEW_NUM || m->view_table[view]) return -1; + const PureBlock *block = &m->view_code[view]; + const PureOp *code = &m->code[block->start]; + return block->length == 2 && code[0].op == PURE_OP_INDEX_LOAD && code[1].op == PURE_OP_INDEX_NUM ? (int)code[0].arg : -1; +} + +int pure_module_text_state(const PureModule *m, uint32_t view) { + if (view >= m->views || m->view_type[view] != PURE_VIEW_TEXT || m->view_table[view]) return -1; + const PureBlock *block = &m->view_code[view]; + return block->length == 1 && m->code[block->start].op == PURE_OP_TEXT_LOAD ? (int)m->code[block->start].arg : -1; +} + +int pure_module_column(const PureModule *m, uint32_t table, const char *name) { + for (uint32_t i = 0; i < m->views; ++i) + if (m->view_table[i] == table + 1 && !strcmp(m->view_names[i], name)) return (int)i; + return -1; +} + +/* ---- Execution ------------------------------------------------------ */ + +/* IEEE 754-2019 minimum/maximum on finite values: -0 orders below +0. */ +static double minimum(double a, double b) { return a < b || (a == b && signbit(a)) ? a : b; } +static double maximum(double a, double b) { return a > b || (a == b && !signbit(a)) ? a : b; } + +typedef struct { double num; PureBytes bytes; } Slot; +typedef struct { + PureApp *app; + const PureInputShape *shape; /* message, or NULL */ + const char *key; /* a keyed message's key */ + int64_t base; /* the revision a message's edit was made against, or -1 */ + PureArena *arena; + double dt; + size_t row; /* the row a column is evaluated for */ + bool modified, exists; + uint32_t (*spans)[2]; /* a message's variable fields: start and size in the payload */ +} Event; + +static PureResult fault(PureApp *app, PureFault kind, uint32_t pc) { + app->fault = kind; + app->fault_pc = pc; + return kind == PURE_FAULT_LIMIT ? PURE_LIMIT : kind == PURE_FAULT_NOT_FOUND ? PURE_NOT_FOUND : + kind == PURE_FAULT_EXISTS ? PURE_CONFLICT : PURE_INVALID; +} + +static bool reserve(PureArena *arena, size_t more) { + if (more <= arena->capacity - arena->used) return true; + size_t capacity = arena->capacity ? arena->capacity : 256; + while (capacity - arena->used < more) { + if (capacity > SIZE_MAX / 2) return false; + capacity *= 2; + } + unsigned char *bytes = realloc(arena->bytes, capacity); + if (!bytes) return false; + arena->bytes = bytes; + arena->capacity = capacity; + return true; +} + +/* Visit a value's bytes in order, from wherever they live. */ +static PureResult visit_value(const PureApp *app, const PureArena *arena, PureBytes v, + bool (*visit)(void *, PureSpan), void *context) { + if (!v.length) return PURE_OK; + const unsigned char *direct = NULL; + switch (v.source) { + case FROM_PAYLOAD: direct = app->payload.data + v.offset; break; + case FROM_COPY: direct = app->copied + v.offset; break; + case FROM_CONSTANT: direct = app->module->constant_data + v.offset; break; + case FROM_ARENA: direct = arena->bytes + v.offset; break; + case FROM_TEXT: + return pure_draft_read_version(app->texts[v.owner], v.version, (size_t)v.offset, (size_t)v.length, + visit, context); + default: + return app->data.read(app->data.context, app->document, v.version, (size_t)v.offset, + (size_t)v.length, visit, context); + } + visit(context, (PureSpan){direct, (size_t)v.length}); + return PURE_OK; +} + +static bool append_span(void *context, PureSpan span) { + PureArena *arena = context; + memmove(arena->bytes + arena->used, span.data, span.size); + arena->used += span.size; + return true; +} + +/* Copy a value to the end of the arena. Room is reserved first, so reading + * an arena value while appending never follows a moved buffer. */ +static PureResult append_value(const PureApp *app, PureArena *arena, PureBytes v) { + if (!reserve(arena, (size_t)v.length)) return PURE_ALLOCATION; + return visit_value(app, arena, v, append_span, arena); +} + +/* A contiguous span of a value, copying into the arena when it is spread out. */ +static PureResult contiguous(const PureApp *app, PureArena *arena, PureBytes v, PureSpan *out) { + if (v.source == FROM_PAYLOAD || v.source == FROM_COPY || v.source == FROM_CONSTANT) { + const unsigned char *base = v.source == FROM_PAYLOAD ? app->payload.data : + v.source == FROM_COPY ? app->copied : app->module->constant_data; + *out = (PureSpan){v.length ? base + v.offset : NULL, (size_t)v.length}; + return PURE_OK; + } + if (v.source == FROM_ARENA) { + *out = (PureSpan){arena->bytes + v.offset, (size_t)v.length}; + return PURE_OK; + } + size_t start = arena->used; + PureResult result = append_value(app, arena, v); + *out = (PureSpan){arena->bytes + start, (size_t)v.length}; + return result; +} + +typedef struct { size_t count; unsigned unit; unsigned utf8; bool valid; } Measure; + +static bool measure(void *context, PureSpan span) { + Measure *m = context; + if (m->unit == 2) m->valid = m->valid && pure_utf8_step(&m->utf8, span); + else m->count += m->unit ? pure_utf8_newlines(span) : pure_utf8_scalars(span); + return true; +} + +static PureBytes text_of(const PureApp *app, uint32_t state) { + PureVersion version = pure_draft_version(app->texts[state]); + return (PureBytes){FROM_TEXT, (uint16_t)state, version, 0, version.bytes}; +} + +static void move_cursors(const PureModule *m, double *values, uint32_t text, size_t at, + size_t removed, size_t inserted) { + for (uint32_t c = 0; c < m->states; ++c) { + if (m->state_type[c] != PURE_STATE_CURSOR || m->state_ref[c] != text) continue; + double position = values[c]; + if (removed) { + if (position >= at + removed) position -= (double)removed; + else if (position > at) position = (double)at; + } else if (position > at || (position == at && !(m->state_flags[c] & PURE_CURSOR_STAYS))) + position += (double)inserted; + values[c] = position; + } +} + +/* Indices follow their item across an insert or remove; an index whose item + * was removed names the next item (or the last, or 0 in an empty list). */ +static void move_indices(const PureModule *m, double *values, uint32_t list, size_t count, + size_t at, bool inserted) { + double last = count ? (double)(count - 1) : 0; + for (uint32_t i = 0; i < m->states; ++i) { + if (m->state_type[i] != PURE_STATE_INDEX || m->state_ref[i] != list) continue; + if (inserted) { + if (count > 1 && values[i] >= (double)at) ++values[i]; + } else { + if (values[i] > (double)at) --values[i]; + values[i] = minimum(values[i], last); + } + } +} + +static bool text_field(const PureModule *m, uint32_t list, uint32_t field) { + return m->record_field_text[m->record[m->state_ref[list]].first + field]; +} + +/* Append bytes to the arena as a value of it. */ +static PureResult keep_bytes(PureArena *arena, const void *bytes, size_t length, PureBytes *out) { + if (!reserve(arena, length)) return PURE_ALLOCATION; + if (length) memcpy(arena->bytes + arena->used, bytes, length); + *out = (PureBytes){FROM_ARENA, 0, {0, 0}, arena->used, length}; + arena->used += length; + return PURE_OK; +} + +/* Run a verified block. Types and stack bounds were proven at load. A change + * the data refuses (a size limit, a missing item) fails the event with its + * fault; other failures pass through. */ +typedef struct { unsigned char *at; } Copy; + +static bool copy_out(void *context, PureSpan span) { + Copy *copy = context; + if (span.size) memcpy(copy->at, span.data, span.size); + copy->at += span.size; + return true; +} + +/* An output's message from its field values, laid out as a message's + * payload (size bytes), numbered and kept with the event's changes. */ +static PureResult send(PureApp *app, uint32_t output, const Slot *fields, const PureArena *arena, size_t size) { + const PureModule *m = app->module; + const PureInputShape *shape = &m->input[output]; + if (app->sent_count == app->sent_capacity) { + size_t capacity = app->sent_capacity ? 2 * app->sent_capacity : 16; + struct PureSent *grown = realloc(app->sent, capacity * sizeof(*grown)); + if (!grown) return PURE_ALLOCATION; + app->sent = grown; + app->sent_capacity = capacity; + } + unsigned char *payload = malloc(size ? size : 1); + if (!payload) return PURE_ALLOCATION; + Copy copy = {payload + shape->prefix}; + for (uint32_t f = 0; f < shape->field_count; ++f) { + const PureField *field = &m->field[shape->field_start + f]; + if (field->type == PURE_FIELD_U32) write32(payload + field->offset, (uint32_t)fields[f].num); + else if (field->type == PURE_FIELD_NUMBER) { + uint64_t bits; + memcpy(&bits, &fields[f].num, 8); + write32(payload + field->offset, (uint32_t)bits); + write32(payload + field->offset + 4, (uint32_t)(bits >> 32)); + } else { + if (field->offset != PURE_FIELD_REST) write32(payload + field->offset, (uint32_t)fields[f].bytes.length); + PureResult result = visit_value(app, arena, fields[f].bytes, copy_out, ©); + if (result != PURE_OK) { free(payload); return result; } + } + } + app->sent[app->sent_count++] = (struct PureSent){output, ++app->last_sent[output], payload, size, false}; + app->links_changed = true; + return PURE_OK; +} + +static PureResult run(const PureModule *m, double *values, PureBlock block, Event *ev, Slot *result) { + Slot stack[PURE_MODULE_STACK]; + size_t sp = 0; + PureApp *app = ev->app; + PureArena *arena = ev->arena; + PureResult status; + PureFault kind; + const PureOp *op = NULL; + for (uint32_t pc = block.start, end = block.start + block.length; pc < end;) { + op = &m->code[pc++]; + uint32_t arg = op->arg; + Slot *top = sp ? &stack[sp - 1] : stack; + switch (op->op) { + case PURE_OP_CONST: stack[sp++].num = op->value; continue; + case PURE_OP_LOAD: stack[sp++].num = values[arg]; continue; + case PURE_OP_DT: stack[sp++].num = ev->dt; continue; + case PURE_OP_NOW: stack[sp++].num = app->now; continue; + case PURE_OP_FLOOR: top->num = floor(top->num); continue; + case PURE_OP_STORE: case PURE_OP_CUR_STORE: values[arg] = stack[--sp].num; continue; + case PURE_OP_JZ: if (stack[--sp].num == 0) pc = arg; continue; + case PURE_OP_JMP: pc = arg; continue; + case PURE_OP_NEG: top->num = -top->num; continue; + case PURE_OP_NOT: top->num = top->num == 0; continue; + case PURE_OP_CLAMP: { + double high = stack[--sp].num, low = stack[--sp].num; + if (low > high) { kind = PURE_FAULT_CLAMP; goto failed; } + stack[sp - 1].num = maximum(low, minimum(stack[sp - 1].num, high)); + continue; + } + case PURE_OP_FIELD: { + const PureField *field = &m->field[ev->shape->field_start + arg]; + if (field->type == PURE_FIELD_U32) stack[sp++].num = read32(app->payload.data + field->offset); + else if (field->type == PURE_FIELD_NUMBER) stack[sp++].num = read_double(app->payload.data + field->offset); + else stack[sp++].bytes = (PureBytes){FROM_PAYLOAD, 0, {0, 0}, ev->spans[arg][0], ev->spans[arg][1]}; + continue; + } + case PURE_OP_DOC: case PURE_OP_DOC_SIZE: { + if (op->op == PURE_OP_DOC && !ev->exists) { kind = PURE_FAULT_NOT_FOUND; goto failed; } + PureVersion version = app->data.version(app->data.context, app->document); + if (op->op == PURE_OP_DOC) stack[sp++].bytes = (PureBytes){FROM_DOCUMENT, 0, version, 0, version.bytes}; + else stack[sp++].num = (double)version.bytes; + continue; + } + case PURE_OP_LEN: case PURE_OP_TEXT_BYTES: top->num = (double)top->bytes.length; continue; + case PURE_OP_REPLACE: { + PureBytes insert = stack[--sp].bytes; + double last = stack[--sp].num, first = stack[--sp].num; + double size = (double)app->data.version(app->data.context, app->document).bytes; + if (first != floor(first) || last != floor(last) || first < 0 || last < first || last > size) { + kind = PURE_FAULT_LIMIT; + goto failed; + } + PureSpan span; + size_t mark = arena->used; + if ((status = contiguous(app, arena, insert, &span)) == PURE_OK) + status = app->data.splice(app->data.context, app->document, (size_t)first, (size_t)(last - first), span); + arena->used = mark; + if (status != PURE_OK) goto refused; + ev->modified = ev->exists = true; + continue; + } + /* A reply outlives the arena mark of later statements: keep it by value. */ + case PURE_OP_REPLY: app->reply = stack[--sp].bytes; app->has_reply = true; continue; + case PURE_OP_REQUIRE: if (stack[--sp].num == 0) { kind = PURE_FAULT_LIMIT; goto failed; } continue; + case PURE_OP_TEXT_CONST: + stack[sp++].bytes = (PureBytes){FROM_CONSTANT, 0, {0, 0}, m->constant_offset[arg], m->constant_length[arg]}; + continue; + case PURE_OP_TEXT_LOAD: stack[sp++].bytes = text_of(app, arg); continue; + case PURE_OP_TEXT_INPUT: stack[sp++].bytes = (PureBytes){FROM_PAYLOAD, 0, {0, 0}, 0, app->payload.size}; continue; + case PURE_OP_NUM_TEXT: { + char digits[32]; + if ((status = keep_bytes(arena, digits, pure_format_number(top->num, digits), &top->bytes)) != PURE_OK) + return status; + continue; + } + case PURE_OP_CONCAT: { + PureBytes right = stack[--sp].bytes, left = stack[sp - 1].bytes; + size_t start = arena->used; + if ((status = append_value(app, arena, left)) != PURE_OK || + (status = append_value(app, arena, right)) != PURE_OK) return status; + stack[sp - 1].bytes = (PureBytes){FROM_ARENA, 0, {0, 0}, start, left.length + right.length}; + continue; + } + case PURE_OP_CHARS: case PURE_OP_LINES: case PURE_OP_TEXT_FROM: { + Measure count = {0, op->op == PURE_OP_LINES ? 1 : op->op == PURE_OP_TEXT_FROM ? 2 : 0, 0, true}; + if ((status = visit_value(app, arena, top->bytes, measure, &count)) != PURE_OK) return status; + if (op->op != PURE_OP_TEXT_FROM) top->num = (double)(count.count + (op->op == PURE_OP_LINES)); + else if (!count.valid || !pure_utf8_complete(count.utf8)) { kind = PURE_FAULT_ENCODING; goto failed; } + continue; + } + /* Positions: a cursor or an index, with the text or list it is in. */ + case PURE_OP_CUR_LOAD: case PURE_OP_INDEX_LOAD: + stack[sp].num = values[arg]; + stack[sp++].bytes.owner = m->state_ref[arg]; + continue; + case PURE_OP_CUR_START: case PURE_OP_CUR_END: case PURE_OP_INDEX_FIRST: case PURE_OP_INDEX_LAST: + case PURE_OP_ROW: { + size_t last = op->op == PURE_OP_CUR_END ? pure_draft_size(app->texts[arg]) : + op->op == PURE_OP_INDEX_LAST ? pure_list_count(app->lists[arg]) : 0; + stack[sp].num = op->op == PURE_OP_ROW ? (double)ev->row : + op->op == PURE_OP_INDEX_LAST && last ? (double)(last - 1) : (double)last; + stack[sp++].bytes.owner = (uint16_t)arg; + continue; + } + case PURE_OP_CUR_MOVE: case PURE_OP_INDEX_MOVE: { + double count = stack[--sp].num; + Slot *at = &stack[sp - 1]; + if (count != floor(count)) { kind = PURE_FAULT_LIMIT; goto failed; } + if (op->op == PURE_OP_INDEX_MOVE) { + size_t items = pure_list_count(app->lists[at->bytes.owner]); + at->num = maximum(0, minimum(at->num + count, items ? (double)(items - 1) : 0)); + continue; + } + PureDraft *text = app->texts[at->bytes.owner]; + PureVersion version = pure_draft_version(text); + at->num = (double)(arg == PURE_UNIT_CHAR ? pure_text_step(text, version, (size_t)at->num, count) + : pure_text_line_step(text, version, (size_t)at->num, count)); + continue; + } + case PURE_OP_CUR_COUNT: { + PureDraft *text = app->texts[top->bytes.owner]; + top->num = (double)pure_text_count(text, pure_draft_version(text), (size_t)top->num, arg); + continue; + } + case PURE_OP_TEXT_INSERT: case PURE_OP_TEXT_SET: { + PureBytes insert = stack[--sp].bytes; + size_t at = op->op == PURE_OP_TEXT_INSERT ? (size_t)stack[--sp].num : 0; + size_t removed = op->op == PURE_OP_TEXT_SET ? pure_draft_size(app->texts[arg]) : 0; + PureSpan span; + size_t mark = arena->used; + if ((status = contiguous(app, arena, insert, &span)) == PURE_OK) + status = pure_draft_splice(app->texts[arg], at, removed, span); + arena->used = mark; + if (status != PURE_OK) goto refused; + if (op->op == PURE_OP_TEXT_SET) { + for (uint32_t c = 0; c < m->states; ++c) + if (m->state_type[c] == PURE_STATE_CURSOR && m->state_ref[c] == arg) values[c] = 0; + } else move_cursors(m, values, arg, at, 0, span.size); + continue; + } + case PURE_OP_TEXT_REMOVE: { + double y = stack[--sp].num, x = stack[--sp].num; + size_t first = (size_t)minimum(x, y), last = (size_t)maximum(x, y); + if ((status = pure_draft_splice(app->texts[arg], first, last - first, (PureSpan){0})) != PURE_OK) goto refused; + if (last > first) move_cursors(m, values, arg, first, last - first, 0); + continue; + } + case PURE_OP_TEXT_LINE: { + PureDraft *text = app->texts[arg]; + PureVersion version = pure_draft_version(text); + size_t first = pure_text_line_start(text, version, (size_t)top->num); + size_t last = pure_text_line_end(text, version, (size_t)top->num); + top->bytes = (PureBytes){FROM_TEXT, (uint16_t)arg, version, first, last - first}; + continue; + } + case PURE_OP_BYTES_FROM: case PURE_OP_INDEX_NUM: continue; /* the same bytes or position, retyped */ + case PURE_OP_LIST_COUNT: stack[sp++].num = (double)pure_list_count(app->lists[arg]); continue; + case PURE_OP_SEND: { + const PureInputShape *output = &m->input[arg]; + size_t base = sp - output->field_count, size = output->prefix; + sp = base; + for (uint32_t f = 0; f < output->field_count; ++f) { + uint8_t type = m->field[output->field_start + f].type; + double number = stack[base + f].num; + if (type == PURE_FIELD_BYTES || type == PURE_FIELD_TEXT) size += (size_t)stack[base + f].bytes.length; + else if (type == PURE_FIELD_U32 && !(number >= 0 && number <= UINT32_MAX && number == floor(number))) { + kind = PURE_FAULT_LIMIT; + goto failed; + } + } + if (size > UINT32_MAX) { kind = PURE_FAULT_LIMIT; goto failed; } + if ((status = send(app, arg, stack + base, arena, size)) != PURE_OK) return status; + continue; + } + case PURE_OP_LIST_APPEND: case PURE_OP_LIST_INSERT: { + PureList *list = app->lists[arg]; + const PureRecord *record = &m->record[m->state_ref[arg]]; + size_t base = sp - record->count, count = pure_list_count(list), at = count; + sp = base; + if (op->op == PURE_OP_LIST_INSERT && (at = (size_t)stack[--sp].num) > count) { + kind = PURE_FAULT_NOT_FOUND; + goto failed; + } + /* Room for every text first, so no span moves while the next is copied. */ + size_t mark = arena->used, total = 0; + for (unsigned f = 0; f < record->count; ++f) + if (text_field(m, arg, f)) total += (size_t)stack[base + f].bytes.length; + if (!reserve(arena, total)) return PURE_ALLOCATION; + PureCell cells[PURE_RECORD_FIELDS]; + for (unsigned f = 0; f < record->count; ++f) { + PureSpan span = {0}; + if (text_field(m, arg, f) && (status = contiguous(app, arena, stack[base + f].bytes, &span)) != PURE_OK) + return status; + cells[f] = (PureCell){stack[base + f].num, (unsigned char *)span.data, span.size}; + } + if (m->state_key[arg]) { /* a keyed list places the item by its key */ + bool found; + unsigned key = m->state_key[arg] - 1u; + at = pure_list_seek(list, key, cells[key], &found); + if (found) { arena->used = mark; kind = PURE_FAULT_EXISTS; goto failed; } + } + status = pure_list_insert(list, at, cells, &app->undo); + arena->used = mark; + if (status != PURE_OK) goto refused; + move_indices(m, values, arg, pure_list_count(list), at, true); + continue; + } + case PURE_OP_LIST_REMOVE: { + size_t at = (size_t)stack[--sp].num; + if ((status = pure_list_remove(app->lists[arg], at, &app->undo)) != PURE_OK) goto refused; + move_indices(m, values, arg, pure_list_count(app->lists[arg]), at, false); + continue; + } + case PURE_OP_ITEM_GET: { + uint32_t list = list_of(arg), field = item_field(arg); + size_t at = (size_t)top->num; + if (at >= pure_list_count(app->lists[list])) { kind = PURE_FAULT_NOT_FOUND; goto failed; } + PureCell cell = pure_list_get(app->lists[list], at, field); + /* A text is copied: the item may change or go before the value is used. */ + if (!text_field(m, list, field)) top->num = cell.num; + else if ((status = keep_bytes(arena, cell.text, cell.size, &top->bytes)) != PURE_OK) return status; + continue; + } + case PURE_OP_ITEM_SET: { + uint32_t list = list_of(arg), field = item_field(arg); + Slot value = stack[--sp]; + size_t at = (size_t)stack[--sp].num, mark = arena->used; + PureSpan span = {0}; + status = text_field(m, list, field) ? contiguous(app, arena, value.bytes, &span) : PURE_OK; + if (status == PURE_OK) + status = pure_list_set(app->lists[list], at, field, + (PureCell){value.num, (unsigned char *)span.data, span.size}, &app->undo); + arena->used = mark; + if (status != PURE_OK) goto refused; + continue; + } + case PURE_OP_INDEX_STORE: { + /* An index from before a remove may be past the end. */ + double position = stack[--sp].num; + if (position && position >= (double)pure_list_count(app->lists[m->state_ref[arg]])) { + kind = PURE_FAULT_NOT_FOUND; + goto failed; + } + values[arg] = position; + continue; + } + case PURE_OP_FOLD: { + /* The block that follows runs once per item, as a column of the list. */ + unsigned how = arg & 0xFF; + PureBlock inner = {pc, arg >> 16}; + PureList *list = app->lists[arg >> 8 & 0xFF]; + pc += inner.length; + double total = 0; + bool any = false; + Event per = *ev; + for (size_t row = 0, items = pure_list_count(list); row < items; ++row) { + Slot out[2]; + size_t mark = arena->used; + per.row = row; + status = run(m, values, inner, &per, out); + arena->used = mark; + if (status != PURE_OK) return status; + double value = out[0].num; + if (how == PURE_FOLD_COUNT || how == PURE_FOLD_FIRST ? value == 0 : out[1].num == 0) continue; + if (how == PURE_FOLD_FIRST) { total = (double)row; any = true; break; } + total = how == PURE_FOLD_COUNT ? total + 1 : how == PURE_FOLD_SUM ? total + value : + !any ? value : how == PURE_FOLD_MAX ? maximum(total, value) : minimum(total, value); + any = true; + } + if (!any && how >= PURE_FOLD_MAX) { kind = PURE_FAULT_NOT_FOUND; goto failed; } + if (!isfinite(total)) { kind = PURE_FAULT_NON_FINITE; goto failed; } + stack[sp].num = total; + stack[sp++].bytes.owner = (uint16_t)(arg >> 8 & 0xFF); /* first: the list the index is in */ + continue; + } + case PURE_OP_TEXT_COMPARE: { + /* Texts order by their bytes: a shorter text before a longer one it begins. */ + PureBytes right = stack[--sp].bytes, left = stack[sp - 1].bytes; + size_t mark = arena->used; + PureSpan a, b; + if (!reserve(arena, (size_t)(left.length + right.length))) return PURE_ALLOCATION; + if ((status = contiguous(app, arena, left, &a)) != PURE_OK || + (status = contiguous(app, arena, right, &b)) != PURE_OK) return status; + size_t shorter = a.size < b.size ? a.size : b.size; + int order = shorter ? memcmp(a.data, b.data, shorter) : 0; + if (!order) order = (a.size > b.size) - (a.size < b.size); + arena->used = mark; + stack[sp - 1].num = arg == PURE_LESS ? order < 0 : arg == PURE_LESS_EQUAL ? order <= 0 : + arg == PURE_GREATER ? order > 0 : arg == PURE_GREATER_EQUAL ? order >= 0 : + arg == PURE_EQUAL ? order == 0 : order != 0; + continue; + } + case PURE_OP_LIST_FIND: case PURE_OP_LIST_HAS: { + /* A keyed list's item by key, found by halving: FIND gives its index, HAS whether it is there. */ + unsigned key = m->state_key[arg] - 1u; + size_t mark = arena->used; + PureCell cell = {top->num, NULL, 0}; + if (text_field(m, arg, key)) { + PureSpan span; + if ((status = contiguous(app, arena, top->bytes, &span)) != PURE_OK) return status; + cell = (PureCell){0, (unsigned char *)span.data, span.size}; + } + bool found; + size_t at = pure_list_seek(app->lists[arg], key, cell, &found); + arena->used = mark; + if (op->op == PURE_OP_LIST_HAS) { top->num = found; continue; } + if (!found) { kind = PURE_FAULT_NOT_FOUND; goto failed; } + top->num = (double)at; + top->bytes.owner = (uint16_t)arg; + continue; + } + case PURE_OP_PICK: stack[sp] = stack[sp - 1 - arg]; ++sp; continue; + case PURE_OP_SLIDE: stack[sp - 1 - arg] = stack[sp - 1]; sp -= arg; continue; + default: break; + } + double b = stack[--sp].num, a = stack[sp - 1].num, value; + switch (op->op) { + case PURE_OP_ADD: value = a + b; break; + case PURE_OP_SUB: value = a - b; break; + case PURE_OP_MUL: value = a * b; break; + case PURE_OP_DIV: + if (b == 0) { kind = PURE_FAULT_DIVIDE; goto failed; } + value = a / b; + break; + case PURE_OP_LT: value = a < b; break; + case PURE_OP_LE: value = a <= b; break; + case PURE_OP_GT: value = a > b; break; + case PURE_OP_GE: value = a >= b; break; + case PURE_OP_EQ: value = a == b; break; + case PURE_OP_NE: value = a != b; break; + case PURE_OP_AND: value = a != 0 && b != 0; break; + case PURE_OP_OR: value = a != 0 || b != 0; break; + case PURE_OP_MIN: value = minimum(a, b); break; + case PURE_OP_MOD: /* the divisor's sign, as floor division leaves it (a zero too) */ + if (b == 0) { kind = PURE_FAULT_DIVIDE; goto failed; } + value = fmod(a, b); + if (value != 0 && (value < 0) != (b < 0)) value += b; + else if (value == 0) value = copysign(0.0, b); + break; + default: value = maximum(a, b); break; + } + if (!isfinite(value)) { kind = PURE_FAULT_NON_FINITE; goto failed; } + stack[sp - 1].num = value; + } + if (result) memcpy(result, stack, sp * sizeof(*stack)); /* a view's value, or a fold block's numbers */ + return PURE_OK; +refused: + kind = status == PURE_LIMIT ? PURE_FAULT_LIMIT : status == PURE_NOT_FOUND ? PURE_FAULT_NOT_FOUND : PURE_FAULT_NONE; + if (kind == PURE_FAULT_NONE) return status; +failed: + return fault(app, kind, (uint32_t)(op - m->code)); +} + +PURE_COLD static PureResult new_text(PureDraft **out, const PureModule *m, uint32_t constant) { + *out = pure_draft_new(); + if (!*out) return PURE_ALLOCATION; + PureSpan initial = {m->constant_data + m->constant_offset[constant], m->constant_length[constant]}; + PureResult result = pure_draft_splice(*out, 0, 0, initial); + if (result != PURE_OK) { pure_draft_close(*out); *out = NULL; } + return result; +} + +PURE_COLD static PureList *new_list(const PureModule *m, uint32_t record) { + return pure_list_new(m->record[record].count, m->record_field_text + m->record[record].first); +} + +static bool pending(const PureApp *app, const void *document) { + for (unsigned i = 0; i < app->pending_count; ++i) if (app->pending[i].document == document) return true; + return false; +} + +void pure_app_finish(PureApp *app) { + /* A document in the unsaved burst stays open until the save. */ + if (app->document && !pending(app, app->document)) app->data.close(app->data.context, app->document); + free(app->copied); + app->document = NULL; + app->copied = NULL; + app->has_reply = false; + app->payload = (PureSpan){0}; + app->arena.used = 0; +} + +/* The start of a burst: what a failed save returns to. Text and stored list + * drafts stay held until the save, so every version since stays readable. */ +static void open_burst(PureApp *app) { + if (app->open) return; + const PureModule *m = app->module; + memcpy(app->durable_values, app->values, sizeof(app->values)); + app->durable_ticks = app->ticks; + app->durable_events = app->events; + for (uint32_t i = 0; i < m->states; ++i) { + if (app->texts[i]) { + pure_draft_hold(app->texts[i], true); + app->durable_texts[i] = pure_draft_version(app->texts[i]); + } + if (app->list_drafts[i]) { + pure_draft_hold(app->list_drafts[i], true); + app->durable_lists[i] = pure_draft_version(app->list_drafts[i]); + } + } + app->open = true; +} + +/* End a burst, keeping it (saved) or undoing all of it. The documents it held + * close, except the current event's, which pure_app_finish closes. */ +static void close_burst(PureApp *app, bool keep) { + for (unsigned i = 0; i < app->pending_count; ++i) + if (app->pending[i].document != app->document) app->data.close(app->data.context, app->pending[i].document); + app->pending_count = 0; + if (!app->open) return; + const PureModule *m = app->module; + for (uint32_t i = 0; i < m->states; ++i) { + if (app->texts[i]) { + if (!keep) pure_draft_restore(app->texts[i], app->durable_texts[i]); + pure_draft_hold(app->texts[i], false); + } + if (app->list_drafts[i]) { + if (!keep) pure_draft_restore(app->list_drafts[i], app->durable_lists[i]); + pure_draft_hold(app->list_drafts[i], false); + } + } + if (keep) pure_undo_commit(&app->undo); + else { + pure_undo_rollback(&app->undo, 0); + memcpy(app->values, app->durable_values, sizeof(app->values)); + app->ticks = app->durable_ticks; + app->events = app->durable_events; + ++app->changes; + } + app->open = false; +} + +static void drop_sent(PureApp *app, size_t count) { + while (app->sent_count > count) free(app->sent[--app->sent_count].payload); +} + +/* After a save: what it stored is the links' saved state (trimmed sends go), + * or, when it failed, the links return to the last saved state. */ +static void settle_links(PureApp *app, bool saved) { + if (saved) { + size_t kept = 0; + for (size_t i = 0; i < app->sent_count; ++i) { + if (app->sent[i].trimmed) free(app->sent[i].payload); + else app->sent[kept++] = app->sent[i]; + } + app->sent_count = app->sent_durable = kept; + memcpy(app->saved_sent, app->last_sent, sizeof(app->saved_sent)); + for (unsigned i = 0; i < app->heard_count; ++i) app->heard[i].saved = app->heard[i].seq; + } else { + drop_sent(app, app->sent_durable); + for (size_t i = 0; i < app->sent_count; ++i) app->sent[i].trimmed = false; + memcpy(app->last_sent, app->saved_sent, sizeof(app->last_sent)); + unsigned kept = 0; + for (unsigned i = 0; i < app->heard_count; ++i) + if (app->heard[i].saved) { + app->heard[kept] = app->heard[i]; + app->heard[kept++].seq = app->heard[i].saved; + } + app->heard_count = kept; + } + app->links_changed = false; +} + +PureResult pure_app_save(PureApp *app) { + void *documents[PURE_APP_PENDING]; + int64_t revisions[PURE_APP_PENDING]; + size_t count = 0; + for (unsigned i = 0; i < app->pending_count; ++i) + if (app->pending[i].edited) documents[count++] = app->pending[i].document; + PureResult result = pure_save_commit(app, app->values, documents, count, revisions); + settle_links(app, result == PURE_OK); + close_burst(app, result == PURE_OK); + return result; +} + +bool pure_app_defer(PureApp *app, bool deferred) { + if (deferred && !pure_save_enabled(app)) return false; + app->deferred = deferred; + return true; +} + +/* ---- Snapshots. Snapshots of one app form a chain from oldest to newest, + * each holding the next, so they go oldest first. List items the app lets go + * while a snapshot is its newest may be read by that one and older ones, so + * they go with it. */ +struct PureSnapshot { + unsigned refs; + const PureModule *module; + uint64_t changes; + double values[PURE_MODULE_STATES]; + PureDraft *texts[PURE_MODULE_STATES]; + PureList *lists[PURE_MODULE_STATES]; + PureRetired retired; + struct PureSnapshot *newer; +}; + +void pure_snapshot_hold(PureSnapshot *snapshot) { __atomic_add_fetch(&snapshot->refs, 1, __ATOMIC_RELAXED); } + +void pure_snapshot_release(PureSnapshot *snapshot) { + while (snapshot && !__atomic_sub_fetch(&snapshot->refs, 1, __ATOMIC_ACQ_REL)) { + PureSnapshot *newer = snapshot->newer; + for (uint32_t i = 0; i < PURE_MODULE_STATES; ++i) { + pure_draft_close(snapshot->texts[i]); + pure_list_free(snapshot->lists[i]); + } + pure_retired_free(&snapshot->retired); + free(snapshot); + snapshot = newer; + } +} + +void pure_app_stop(PureApp *app) { + pure_app_finish(app); + close_burst(app, true); /* unsaved changes are dropped with the app */ + for (uint32_t i = 0; i < PURE_MODULE_STATES && !app->reading; ++i) { + pure_draft_close(app->texts[i]); + pure_list_free(app->lists[i]); + pure_draft_close(app->list_drafts[i]); + } + /* Items the newest snapshot may still read go when it goes. */ + if (app->snapshot) { + app->snapshot->retired = app->retired; + pure_snapshot_release(app->snapshot); + } else + pure_retired_free(&app->retired); + pure_draft_close(app->stored_values); + drop_sent(app, 0); + free(app->sent); + free(app->heard); + free(app->carried); + pure_draft_close(app->stored_links); + free(app->pending); + pure_undo_free(&app->undo); + for (unsigned i = 0; i < app->row_tables; ++i) free(app->rows[i].items); + free(app->rows); + free(app->arena.bytes); + free(app->view_arena.bytes); + memset(app, 0, sizeof(*app)); +} + +/* Where one event began inside the burst: a failed event returns there. */ +typedef struct { + PureVersion texts[PURE_MODULE_STATES], lists[PURE_MODULE_STATES]; + size_t undo, sent; + uint64_t last_sent[PURE_MODULE_INPUTS]; + bool opened; +} Mark; + +static void mark_event(PureApp *app, Mark *mark) { + mark->opened = !app->open; + open_burst(app); + for (uint32_t i = 0; i < app->module->states; ++i) { + if (app->texts[i]) mark->texts[i] = pure_draft_version(app->texts[i]); + if (app->list_drafts[i]) mark->lists[i] = pure_draft_version(app->list_drafts[i]); + } + mark->undo = app->undo.count; + mark->sent = app->sent_count; + memcpy(mark->last_sent, app->last_sent, sizeof(mark->last_sent)); +} + +static void undo_event(PureApp *app, const Mark *mark) { + for (uint32_t i = 0; i < app->module->states; ++i) { + if (app->texts[i]) pure_draft_restore(app->texts[i], mark->texts[i]); + if (app->list_drafts[i]) pure_draft_restore(app->list_drafts[i], mark->lists[i]); + } + pure_undo_rollback(&app->undo, mark->undo); + drop_sent(app, mark->sent); + memcpy(app->last_sent, mark->last_sent, sizeof(app->last_sent)); + if (mark->opened) close_burst(app, true); /* nothing happened: release the holds */ +} + +/* The last number heard from source, or NULL. */ +static struct PureHeard *heard_from(PureApp *app, const char *source) { + for (unsigned i = 0; i < app->heard_count; ++i) + if (!strcmp(app->heard[i].source, source)) return &app->heard[i]; + return NULL; +} + +/* Hear seq from source (room for a new source was made before). */ +static void hear(PureApp *app, const char *source, uint64_t seq) { + struct PureHeard *heard = heard_from(app, source); + if (!heard) { + heard = &app->heard[app->heard_count++]; + *heard = (struct PureHeard){.saved = 0}; + memcpy(heard->source, source, strlen(source) + 1); + } + heard->seq = seq; + app->links_changed = true; +} + +static PureResult keep_reply(PureApp *app); + +/* One event: every handler for it runs in module order against pending + * states, all or nothing. A key, text or message event saves before it + * returns unless saving is deferred; a frame leaves that to the next event + * (frames come 60 times a second). A message's document joins the burst. */ +static PureResult dispatch(PureApp *app, uint32_t event, Event *ev) { + const PureModule *m = app->module; + double pending[PURE_MODULE_STATES]; + Mark mark; + mark_event(app, &mark); + memcpy(pending, app->values, sizeof(app->values)); + app->fault = PURE_FAULT_NONE; + app->fault_pc = UINT32_MAX; + PureResult result = PURE_OK; + for (uint32_t i = 0; i < m->handlers && result == PURE_OK; ++i) + if (m->handler[i].event == event) result = run(m, pending, m->handler[i].code, ev, NULL); + /* An edit needs the revision it was made against. A save rebases an + * edited document and releasing texts may reclaim earlier text versions: + * keep such a reply by copying it first. */ + if (result == PURE_OK && ev->modified && ev->base < 0) result = PURE_INVALID; + if (result == PURE_OK && app->has_reply && + ((app->reply.source == FROM_DOCUMENT && ev->modified) || app->reply.source == FROM_TEXT)) + result = keep_reply(app); + if (result != PURE_OK) { + undo_event(app, &mark); + return result; + } + memcpy(app->values, pending, sizeof(pending)); + ++app->events; + ++app->changes; + if (!event) ++app->ticks; + if (app->document) { + app->pending[app->pending_count].document = app->document; + app->pending[app->pending_count].edited = ev->modified; + memcpy(app->pending[app->pending_count++].key, ev->key, strlen(ev->key) + 1); + app->revision += ev->modified; /* a save moves an edited document one revision */ + } + if (app->delivering.pending) { + hear(app, app->delivering.source, app->delivering.seq); + app->delivering.pending = false; + } + return event && !app->deferred ? pure_app_save(app) : PURE_OK; +} + +PureResult pure_app_frame(PureApp *app) { + pure_app_finish(app); + Event ev = {.app = app, .arena = &app->arena, .dt = 1.0 / 60.0}; + return dispatch(app, 0, &ev); +} + +PureResult pure_app_tick(PureApp *app, uint32_t input) { + if (input >= app->module->inputs || app->module->input[input].kind != PURE_KIND_TIMER) return PURE_INVALID; + pure_app_finish(app); + Event ev = {.app = app, .arena = &app->arena}; + return dispatch(app, 1 + 2 * input, &ev); +} + +PureResult pure_app_input(PureApp *app, uint32_t input, PureEdge edge) { + if (input >= app->module->inputs || app->module->input[input].kind != PURE_KIND_KEY || + (edge != PURE_EDGE_DOWN && edge != PURE_EDGE_UP)) return PURE_INVALID; + pure_app_finish(app); + Event ev = {.app = app, .arena = &app->arena}; + return dispatch(app, 1 + 2 * input + (edge == PURE_EDGE_UP), &ev); +} + +PureResult pure_app_point(PureApp *app, uint32_t state, size_t position) { + const PureModule *m = app->module; + pure_app_finish(app); + if (state >= m->states || m->state_type[state] != PURE_STATE_INDEX) return PURE_INVALID; + if (position >= pure_list_count(app->lists[m->state_ref[state]])) return PURE_NOT_FOUND; + open_burst(app); /* what a failed save returns to */ + app->fault = PURE_FAULT_NONE; + app->fault_pc = UINT32_MAX; + app->values[state] = (double)position; + ++app->events; + ++app->changes; + return app->deferred ? PURE_OK : pure_app_save(app); +} + +PureResult pure_app_text(PureApp *app, uint32_t input, PureSpan text) { + pure_app_finish(app); + if (input >= app->module->inputs || app->module->input[input].kind != PURE_KIND_TEXT || + (text.size && !text.data) || !pure_utf8_valid(text)) return PURE_INVALID; + app->payload = text; + Event ev = {.app = app, .arena = &app->arena}; + PureResult result = dispatch(app, 1 + 2 * input, &ev); + app->payload = (PureSpan){0}; + return result; +} + +static PureResult keep_reply(PureApp *app) { + size_t length = (size_t)app->reply.length, start = app->arena.used; + unsigned char *copy = malloc(length ? length : 1); + if (!copy) return PURE_ALLOCATION; + PureResult result = append_value(app, &app->arena, app->reply); + if (result != PURE_OK) { free(copy); return result; } + if (length) memcpy(copy, app->arena.bytes + start, length); + app->arena.used = start; + app->copied = copy; + app->reply = (PureBytes){FROM_COPY, 0, {0, 0}, 0, length}; + return PURE_OK; +} + +/* A message whose variable fields are where placed says (start and size in + * the payload), or, without it, one after another after the prefix. */ +static PureResult message(PureApp *app, uint32_t input, const char *key, int64_t base, PureSpan payload, + const uint32_t *placed) { + pure_app_finish(app); + const PureModule *m = app->module; + app->fault = PURE_FAULT_NONE; + app->fault_pc = UINT32_MAX; + app->revision = 0; + if (input >= m->inputs || m->input[input].kind != PURE_KIND_MESSAGE || base < -1 || + (payload.size && !payload.data) || (m->input[input].keyed ? !document_key(key) : key != NULL)) + return PURE_INVALID; + const PureInputShape *shape = &m->input[input]; + if (payload.size < shape->prefix) return fault(app, PURE_FAULT_LIMIT, UINT32_MAX); + /* Where each variable field is; a length past the payload, a text that + * is not UTF-8 or a number that is not finite fails the message. */ + uint32_t spans[PURE_MODULE_FIELDS][2]; + size_t at = shape->prefix; + for (uint32_t f = 0; f < shape->field_count; ++f) { + const PureField *field = &m->field[shape->field_start + f]; + if (field->type == PURE_FIELD_NUMBER && !isfinite(read_double(payload.data + field->offset))) + return fault(app, PURE_FAULT_NON_FINITE, UINT32_MAX); + if (field->type != PURE_FIELD_BYTES && field->type != PURE_FIELD_TEXT) continue; + size_t start = placed ? placed[2 * f] : at; + if (start > payload.size) return fault(app, PURE_FAULT_LIMIT, UINT32_MAX); + size_t size = placed ? placed[2 * f + 1] : field->offset == PURE_FIELD_REST ? payload.size - at : + read32(payload.data + field->offset); + if (size > payload.size - start) return fault(app, PURE_FAULT_LIMIT, UINT32_MAX); + if (field->type == PURE_FIELD_TEXT && !pure_utf8_valid((PureSpan){size ? payload.data + start : NULL, size})) + return fault(app, PURE_FAULT_ENCODING, UINT32_MAX); + spans[f][0] = (uint32_t)start; + spans[f][1] = (uint32_t)size; + at = start + size; + } + if (shape->keyed && m->collections) { + if (!app->data.open) return PURE_INVALID; + /* A key joins a burst once, so its revision moves once per save. */ + bool seen = app->pending_count == PURE_APP_PENDING; + for (unsigned i = 0; i < app->pending_count && !seen; ++i) seen = !strcmp(app->pending[i].key, key); + if (seen) { + PureResult saved = pure_app_save(app); + if (saved != PURE_OK) return saved; + } + if (app->pending_count == app->pending_capacity) { + unsigned capacity = app->pending_capacity ? 2 * app->pending_capacity : 16; + struct PurePending *grown = realloc(app->pending, capacity * sizeof(*grown)); + if (!grown) return PURE_ALLOCATION; + app->pending = grown; + app->pending_capacity = capacity; + } + PureResult opened = app->data.open(app->data.context, 0, key, &app->document, &app->revision); + if (opened != PURE_OK) { app->document = NULL; return opened; } + if (base >= 0 && base != app->revision) { pure_app_finish(app); return PURE_CONFLICT; } + } + app->payload = payload; + Event ev = {.app = app, .shape = shape, .key = key, .base = base, .arena = &app->arena, .exists = app->revision > 0, + .spans = spans}; + PureResult result = dispatch(app, 1 + 2 * input, &ev); + if (result != PURE_OK) pure_app_finish(app); + return result; +} + +PureResult pure_app_message(PureApp *app, uint32_t input, const char *key, int64_t base, PureSpan payload) { + return message(app, input, key, base, payload, NULL); +} + +PureResult pure_app_message_fields(PureApp *app, uint32_t input, const char *key, int64_t base, PureSpan payload, + const uint32_t *placed) { + return message(app, input, key, base, payload, placed); +} + +void pure_app_outbox(const PureApp *app, PureSentVisit visit, void *context) { + for (size_t i = 0; i < app->sent_durable; ++i) { + const struct PureSent *sent = &app->sent[i]; + if (!sent->trimmed && !visit(context, sent->output, sent->seq, (PureSpan){sent->payload, sent->size})) return; + } +} + +void pure_app_outbox_newest(const PureApp *app, PureSentVisit visit, void *context) { + for (size_t i = app->sent_durable; i-- > 0;) { + const struct PureSent *sent = &app->sent[i]; + if (!sent->trimmed && !visit(context, sent->output, sent->seq, (PureSpan){sent->payload, sent->size})) return; + } +} + +void pure_app_trim(PureApp *app, uint32_t output, uint64_t through) { + for (size_t i = 0; i < app->sent_durable; ++i) + if (app->sent[i].output == output && app->sent[i].seq <= through && !app->sent[i].trimmed) { + app->sent[i].trimmed = true; + app->links_changed = true; + } +} + +PureResult pure_app_deliver(PureApp *app, uint32_t input, const char *source, uint64_t seq, PureSpan payload, + const uint32_t *placed) { + const PureModule *m = app->module; + size_t length = source ? strlen(source) : 0; + app->duplicate = false; + if (input >= m->inputs || m->input[input].kind != PURE_KIND_MESSAGE || m->input[input].keyed || !length || + length > PURE_SOURCE || !seq) return PURE_INVALID; + for (size_t i = 0; i < length; ++i) + if (source[i] <= ' ' || source[i] > '~') return PURE_INVALID; + struct PureHeard *heard = heard_from(app, source); + if (heard && seq <= heard->seq) { + pure_app_finish(app); + app->duplicate = true; + return PURE_OK; + } + if (!heard && app->heard_count == app->heard_capacity) { + unsigned capacity = app->heard_capacity ? 2 * app->heard_capacity : 4; + struct PureHeard *grown = realloc(app->heard, capacity * sizeof(*grown)); + if (!grown) return PURE_ALLOCATION; + app->heard = grown; + app->heard_capacity = capacity; + } + app->delivering.source = source; + app->delivering.seq = seq; + app->delivering.pending = true; + PureResult result = message(app, input, NULL, -1, payload, placed); + if (!app->delivering.pending) return result; + /* Not applied. Refused by the app (a fault), the message was delivered + * all the same: its number is heard, saved like a key event's changes. */ + app->delivering.pending = false; + if (app->fault == PURE_FAULT_NONE) return result; + Mark mark; + mark_event(app, &mark); + hear(app, source, seq); + PureResult saved = app->deferred ? PURE_OK : pure_app_save(app); + return saved == PURE_OK ? result : saved; +} + +PureResult pure_app_reply(const PureApp *app, bool (*visit)(void *, PureSpan), void *context) { + if (!app->has_reply) return PURE_OK; + return visit_value(app, &app->arena, app->reply, visit, context); +} + +/* A view that fails reports it by its result; the fault stays the last event's. */ +static PureResult evaluate(PureApp *app, uint32_t view, size_t row, Slot *out) { + double values[PURE_MODULE_STATES]; + memcpy(values, app->values, sizeof(values)); + app->view_arena.used = 0; + Event ev = {.app = app, .arena = &app->view_arena, .row = row}; + PureFault fault = app->fault; + uint32_t fault_pc = app->fault_pc; + PureResult result = run(app->module, values, app->module->view_code[view], &ev, out); + app->fault = fault; + app->fault_pc = fault_pc; + return result; +} + +/* A filtered table's items, as fresh as the app's states: the filter runs + * once per item after each change (an item it fails on is left out), not + * once per cell. NULL for a table without a filter, or without room. */ +static struct PureRows *filtered(PureApp *app, uint32_t table) { + const PureModule *m = app->module; + if (!m->view_code[table].length) return NULL; + struct PureRows *rows = NULL; + for (unsigned i = 0; i < app->row_tables && !rows; ++i) if (app->rows[i].table == table) rows = &app->rows[i]; + if (!rows) { + struct PureRows *grown = realloc(app->rows, (app->row_tables + 1) * sizeof(*grown)); + if (!grown) return NULL; + app->rows = grown; + rows = &app->rows[app->row_tables++]; + *rows = (struct PureRows){.table = table, .changes = UINT64_MAX}; + } + if (rows->changes == app->changes) return rows; + rows->count = 0; + for (size_t item = 0, items = pure_list_count(app->lists[m->view_list[table]]); item < items; ++item) { + Slot keep; + if (evaluate(app, table, item, &keep) != PURE_OK || keep.num == 0) continue; + if (rows->count == rows->capacity) { + size_t capacity = rows->capacity ? 2 * rows->capacity : 16; + size_t *grown = realloc(rows->items, capacity * sizeof(*grown)); + if (!grown) return NULL; + rows->items = grown; + rows->capacity = capacity; + } + rows->items[rows->count++] = item; + } + rows->changes = app->changes; + return rows; +} + +bool pure_app_row_item(PureApp *app, uint32_t table, size_t row, size_t *item) { + const PureModule *m = app->module; + if (table >= m->views || m->view_type[table] != PURE_VIEW_TABLE) return false; + if (!m->view_code[table].length) { + *item = row; + return row < pure_list_count(app->lists[m->view_list[table]]); + } + struct PureRows *rows = filtered(app, table); + if (!rows || row >= rows->count) return false; + *item = rows->items[row]; + return true; +} + +bool pure_app_item_row(PureApp *app, uint32_t table, size_t item, size_t *row) { + const PureModule *m = app->module; + if (table >= m->views || m->view_type[table] != PURE_VIEW_TABLE) return false; + if (!m->view_code[table].length) { + *row = item; + return item < pure_list_count(app->lists[m->view_list[table]]); + } + struct PureRows *rows = filtered(app, table); + size_t low = 0, high = rows ? rows->count : 0; /* the items are in order */ + while (low < high) { + size_t middle = low + (high - low) / 2; + if (rows->items[middle] < item) low = middle + 1; + else high = middle; + } + if (low == (rows ? rows->count : 0) || rows->items[low] != item) return false; + *row = low; + return true; +} + +/* A view outside a table (column false), or a column and a row of its table. */ +static PureResult view_of(PureApp *app, uint32_t view, bool column, size_t row, uint8_t type, Slot *slot) { + const PureModule *m = app->module; + if (view >= m->views || m->view_type[view] != type || (m->view_table[view] != 0) != column) return PURE_INVALID; + if (column && !pure_app_row_item(app, m->view_table[view] - 1u, row, &row)) return PURE_NOT_FOUND; + return evaluate(app, view, row, slot); +} + +bool pure_app_view(PureApp *app, uint32_t view, double *out) { + Slot slot; + if (view_of(app, view, false, 0, PURE_VIEW_NUM, &slot) != PURE_OK) return false; + *out = slot.num; + return true; +} + +PureResult pure_app_view_text(PureApp *app, uint32_t view, bool (*visit)(void *, PureSpan), void *context) { + Slot slot; + PureResult result = view_of(app, view, false, 0, PURE_VIEW_TEXT, &slot); + return result == PURE_OK ? visit_value(app, &app->view_arena, slot.bytes, visit, context) : result; +} + +size_t pure_app_rows(PureApp *app, uint32_t table) { + const PureModule *m = app->module; + if (table >= m->views || m->view_type[table] != PURE_VIEW_TABLE) return 0; + struct PureRows *rows = filtered(app, table); + return rows ? rows->count : m->view_code[table].length ? 0 : pure_list_count(app->lists[m->view_list[table]]); +} + +bool pure_app_cell(PureApp *app, uint32_t column, size_t row, double *out) { + Slot slot; + if (view_of(app, column, true, row, PURE_VIEW_NUM, &slot) != PURE_OK) return false; + *out = slot.num; + return true; +} + +PureResult pure_app_cell_text(PureApp *app, uint32_t column, size_t row, bool (*visit)(void *, PureSpan), void *context) { + Slot slot; + PureResult result = view_of(app, column, true, row, PURE_VIEW_TEXT, &slot); + return result == PURE_OK ? visit_value(app, &app->view_arena, slot.bytes, visit, context) : result; +} + +PureSnapshot *pure_app_snapshot(PureApp *app) { + if (app->open || app->reading) return NULL; + const PureModule *m = app->module; + PureSnapshot *made = calloc(1, sizeof(*made)), *last = app->snapshot; + if (!made) return NULL; + made->refs = 1; + made->module = m; + made->changes = app->changes; + memcpy(made->values, app->values, sizeof(made->values)); + bool ok = true; + for (uint32_t i = 0; i < m->states && ok; ++i) { + if (app->texts[i]) ok = (made->texts[i] = pure_draft_freeze(app->texts[i])) != NULL; + if (!ok || !app->lists[i]) continue; + if (last && last->lists[i] && pure_list_current(last->lists[i], app->lists[i])) { + pure_list_hold(last->lists[i]); + made->lists[i] = last->lists[i]; + } else + ok = (made->lists[i] = pure_list_share(app->lists[i], &app->retired)) != NULL; + } + if (!ok) { + pure_snapshot_release(made); + return NULL; + } + made->refs = last ? 3 : 2; /* the app's, the caller's and the last one's */ + if (last) { + last->retired = app->retired; + last->newer = made; + pure_snapshot_release(last); + } else + pure_retired_free(&app->retired); /* shared only by a snapshot never made */ + app->retired = (PureRetired){0}; + app->snapshot = made; + return made; +} + +void pure_app_read(PureApp *reader, const PureSnapshot *snapshot) { + reader->module = snapshot->module; + reader->reading = true; + reader->changes = snapshot->changes; /* what the row cache is fresh for */ + memcpy(reader->values, snapshot->values, sizeof(reader->values)); + memcpy(reader->texts, snapshot->texts, sizeof(reader->texts)); + memcpy(reader->lists, snapshot->lists, sizeof(reader->lists)); +} + +/* A list with record's fields: each takes the field of `from` with the same + * name and kind, or its default. */ +PURE_COLD PureResult pure_list_migrate(PureList **out, const PureList *from, const char *const *names, + const uint8_t *text, unsigned count, const PureModule *m, uint32_t record) { + const PureRecord *r = &m->record[record]; + int source[PURE_RECORD_FIELDS]; + for (unsigned f = 0; f < r->count; ++f) { + source[f] = -1; + for (unsigned k = 0; k < count; ++k) + if (!strcmp(m->record_field_names[r->first + f], names[k]) && m->record_field_text[r->first + f] == text[k]) + source[f] = (int)k; + } + if (!(*out = new_list(m, record))) return PURE_ALLOCATION; + PureResult result = PURE_OK; + for (size_t at = 0; result == PURE_OK && at < pure_list_count(from); ++at) { + PureCell cells[PURE_RECORD_FIELDS]; + for (unsigned f = 0; f < r->count; ++f) { + uint32_t field = r->first + f, constant = (uint32_t)m->record_field_default[field]; + cells[f] = source[f] >= 0 ? pure_list_get(from, at, (unsigned)source[f]) : + m->record_field_text[field] ? + (PureCell){0, (unsigned char *)m->constant_data + m->constant_offset[constant], + m->constant_length[constant]} : + (PureCell){m->record_field_default[field], NULL, 0}; + } + result = pure_list_insert(*out, at, cells, NULL); + } + if (result != PURE_OK) { pure_list_free(*out); *out = NULL; } + return result; +} + +/* A kept list keeps its items; with the same record it is the same list. */ +PURE_COLD static PureResult keep_list(PureList **out, PureList *from, const PureModule *old, uint32_t old_record, + const PureModule *m, uint32_t record) { + const PureRecord *r = &m->record[record], *o = &old->record[old_record]; + const char *names[PURE_RECORD_FIELDS]; + bool same = r->count == o->count; + for (unsigned k = 0; k < o->count; ++k) { + names[k] = old->record_field_names[o->first + k]; + same = same && !strcmp(names[k], m->record_field_names[r->first + k]) && + old->record_field_text[o->first + k] == m->record_field_text[r->first + k]; + } + if (same) { *out = from; return PURE_OK; } + return pure_list_migrate(out, from, names, old->record_field_text + o->first, o->count, m, record); +} + +/* Open a module in an app. States keep their values by name and kind (texts + * their content, cursors their position in it, lists their items, moved by + * field name when the record changed); every other state starts from its + * stored value, or its initial one. A list whose record changed is stored + * again, and the next event or save publishes the module's states; a state + * the module drops stays stored as it was. Starting an app opens its module + * over none. */ +/* Outputs of old that module has by name and with the same fields keep their + * sends and numbers; one it lacks or changed refuses the module while sends + * are left on it (CONFLICT), and otherwise keeps its number for later. */ +PURE_COLD static PureResult remap_outputs(PureApp *app, const PureModule *old, const PureModule *module, bool apply) { + int map[PURE_MODULE_INPUTS]; + for (uint32_t i = 0; i < old->inputs; ++i) { + map[i] = -1; + if (old->input[i].kind != PURE_KIND_OUTPUT) continue; + int to = pure_module_input(module, old->input_names[i]); + bool same = to >= 0 && module->input[to].kind == PURE_KIND_OUTPUT && + module->input[to].field_count == old->input[i].field_count; + for (uint32_t f = 0; same && f < old->input[i].field_count; ++f) + same = module->field[module->input[to].field_start + f].type == old->field[old->input[i].field_start + f].type; + map[i] = same ? to : -1; + for (size_t s = 0; !same && s < app->sent_count; ++s) + if (app->sent[s].output == i) return PURE_CONFLICT; + } + if (!apply) return PURE_OK; + uint64_t last[PURE_MODULE_INPUTS] = {0}, saved[PURE_MODULE_INPUTS] = {0}; + for (uint32_t i = 0; i < old->inputs; ++i) { + if (old->input[i].kind != PURE_KIND_OUTPUT || !app->last_sent[i]) continue; + if (map[i] >= 0) { + last[map[i]] = app->last_sent[i]; + saved[map[i]] = app->saved_sent[i]; + continue; + } + struct PureCarried *grown = realloc(app->carried, (app->carried_count + 1) * sizeof(*grown)); + if (!grown) continue; /* its numbers would start again: kept only while memory allows */ + app->carried = grown; + memcpy(grown[app->carried_count].name, old->input_names[i], PURE_MODULE_NAME); + grown[app->carried_count++].last = app->last_sent[i]; + app->links_changed = true; + } + for (unsigned c = 0; c < app->carried_count;) { + int to = pure_module_input(module, app->carried[c].name); + if (to < 0 || module->input[to].kind != PURE_KIND_OUTPUT || last[to]) { ++c; continue; } + last[to] = saved[to] = app->carried[c].last; + app->carried[c] = app->carried[--app->carried_count]; + app->links_changed = true; + } + for (size_t s = 0; s < app->sent_count; ++s) app->sent[s].output = (uint32_t)map[app->sent[s].output]; + memcpy(app->last_sent, last, sizeof(last)); + memcpy(app->saved_sent, saved, sizeof(saved)); + return PURE_OK; +} + +PURE_COLD static PureResult open_module(PureApp *app, const PureModule *module) { + const PureModule *old = app->module; + bool stored = pure_save_enabled(app); + double values[PURE_MODULE_STATES] = {0}; + PureDraft *texts[PURE_MODULE_STATES] = {0}, *list_drafts[PURE_MODULE_STATES] = {0}; + PureList *lists[PURE_MODULE_STATES] = {0}; + PureVersion saved[PURE_MODULE_STATES] = {{0, 0}}, before[PURE_MODULE_STATES] = {{0, 0}}; + int kept[PURE_MODULE_STATES]; + bool rewrite[PURE_MODULE_STATES] = {false}; + PureResult result = PURE_OK; + for (uint32_t i = 0; i < module->states; ++i) { + int previous = old ? pure_module_state(old, module->state_names[i]) : -1; + kept[i] = previous >= 0 && old->state_type[previous] == module->state_type[i] ? previous : -1; + if (kept[i] >= 0) saved[i] = app->saved[kept[i]]; + if (module->state_type[i] == PURE_STATE_TEXT) { + if (kept[i] >= 0) texts[i] = app->texts[kept[i]]; + else if (result == PURE_OK) + result = stored ? pure_save_open_text(app, module, i, &texts[i], &saved[i]) : + new_text(&texts[i], module, module->state_ref[i]); + } else if (module->state_type[i] == PURE_STATE_LIST) { + if (result != PURE_OK) continue; + if (kept[i] >= 0) { + result = keep_list(&lists[i], app->lists[kept[i]], old, old->state_ref[kept[i]], module, + module->state_ref[i]); + list_drafts[i] = app->list_drafts[kept[i]]; + rewrite[i] = result == PURE_OK && list_drafts[i] && lists[i] != app->lists[kept[i]]; + } else if (stored) { + result = pure_save_open_list(app, module, i, &lists[i], &list_drafts[i], &saved[i]); + } else if (!(lists[i] = new_list(module, module->state_ref[i]))) result = PURE_ALLOCATION; + } else values[i] = kept[i] >= 0 ? app->values[kept[i]] : module->initial[i]; + } + /* A keyed list keeps its items in key order: one that was not keyed, or + * keyed by another field, is put in order and its indices follow their + * items. Two items with one key refuse the module: nothing is dropped. */ + size_t *moved[PURE_MODULE_STATES] = {0}; + for (uint32_t i = 0; i < module->states && result == PURE_OK; ++i) { + if (module->state_type[i] != PURE_STATE_LIST || !module->state_key[i]) continue; + size_t count = pure_list_count(lists[i]); + PureList *sorted = NULL; + if (count && !(moved[i] = malloc(count * sizeof(**moved)))) result = PURE_ALLOCATION; + else result = pure_list_sorted(lists[i], module->state_key[i] - 1u, &sorted, moved[i]); + if (!sorted) { + free(moved[i]); + moved[i] = NULL; + continue; + } + if (kept[i] < 0 || lists[i] != app->lists[kept[i]]) pure_list_free(lists[i]); + lists[i] = sorted; + rewrite[i] = list_drafts[i] != NULL; + } + /* A cursor or index keeps its position only while its text or list is the + * one it belonged to. */ + for (uint32_t i = 0; i < module->states; ++i) { + if (module->state_type[i] != PURE_STATE_CURSOR && module->state_type[i] != PURE_STATE_INDEX) continue; + uint32_t owner = module->state_ref[i]; + bool same = kept[i] >= 0 && kept[owner] >= 0 && old->state_ref[kept[i]] == (unsigned)kept[owner]; + values[i] = same ? app->values[kept[i]] : 0; + } + if (result == PURE_OK && stored) result = pure_save_open_values(app, module, values, kept, texts, lists); + /* Outputs keep their sends and numbers by name. */ + if (result == PURE_OK) result = old ? remap_outputs(app, old, module, false) : stored ? pure_save_open_links(app, module) : PURE_OK; + for (uint32_t i = 0; i < module->states; ++i) { + uint32_t list = module->state_ref[i]; + if (module->state_type[i] == PURE_STATE_INDEX && moved[list] && values[i] < (double)pure_list_count(lists[list])) + values[i] = (double)moved[list][(size_t)values[i]]; + } + for (uint32_t i = 0; i < module->states; ++i) free(moved[i]); + /* Store the lists whose record or order changed last, so that a failure + * can put every draft back as it was. */ + for (uint32_t i = 0; i < module->states; ++i) + if (rewrite[i]) { + before[i] = pure_draft_version(list_drafts[i]); + pure_draft_hold(list_drafts[i], true); + } + for (uint32_t i = 0; i < module->states && result == PURE_OK; ++i) + if (rewrite[i]) result = pure_save_rewrite_list(module, i, lists[i], list_drafts[i]); + for (uint32_t i = 0; i < module->states; ++i) + if (rewrite[i]) { + if (result != PURE_OK) pure_draft_restore(list_drafts[i], before[i]); + pure_draft_hold(list_drafts[i], false); + } + if (result != PURE_OK) { + for (uint32_t i = 0; i < module->states; ++i) { + if (kept[i] >= 0) { + if (lists[i] && lists[i] != app->lists[kept[i]]) pure_list_free(lists[i]); + continue; + } + pure_draft_close(texts[i]); + pure_list_free(lists[i]); + pure_draft_close(list_drafts[i]); + } + return result; + } + for (uint32_t i = 0; old && i < old->states; ++i) { + bool reused = false, draft_reused = false; + for (uint32_t k = 0; k < module->states; ++k) { + reused = reused || (texts[k] && texts[k] == app->texts[i]) || (lists[k] && lists[k] == app->lists[i]); + draft_reused = draft_reused || (list_drafts[k] && list_drafts[k] == app->list_drafts[i]); + } + if (!reused) { + pure_draft_close(app->texts[i]); + pure_list_free(app->lists[i]); + } + if (!draft_reused) pure_draft_close(app->list_drafts[i]); + } + memcpy(app->values, values, sizeof(values)); + memcpy(app->texts, texts, sizeof(texts)); + memcpy(app->lists, lists, sizeof(lists)); + memcpy(app->list_drafts, list_drafts, sizeof(list_drafts)); + memcpy(app->saved, saved, sizeof(saved)); + if (old) remap_outputs(app, old, module, true); + app->module = module; + ++app->changes; + return PURE_OK; +} + +PURE_COLD PureResult pure_app_start(PureApp *app, const PureModule *module, PureData data) { + *app = (PureApp){.data = data, .fault_pc = UINT32_MAX}; + PureResult result = open_module(app, module); + if (result == PURE_OK) result = pure_app_save(app); + if (result != PURE_OK) pure_app_stop(app); + return result; +} + +PURE_COLD PureResult pure_app_replace(PureApp *app, const PureModule *module) { + pure_app_finish(app); + if (app->open || app->pending_count) { + PureResult saved = pure_app_save(app); /* the burst belongs to the module that made it */ + if (saved != PURE_OK) return saved; + } + return open_module(app, module); +} diff --git a/core/module.h b/core/module.h new file mode 100644 index 0000000..f62326b --- /dev/null +++ b/core/module.h @@ -0,0 +1,343 @@ +#ifndef PURE_MODULE_H +#define PURE_MODULE_H +#include "flow.h" +#include "result.h" +#include "module-list.h" +#include "state.h" +#include +#include +#include + +/* Pure module format v5 (docs/design/008-module-format.md). A toolchain such + * as aup produces it; Pure verifies it completely before anything executes: + * operand types (including which text a cursor and which list an index + * belongs to), stack heights and the stack at every join, forward-only + * branches, names, records, tables, constants and effects. */ +enum { + PURE_MODULE_STATES = 32, PURE_MODULE_INPUTS = 32, PURE_MODULE_VIEWS = 64, + PURE_MODULE_HANDLERS = 64, PURE_MODULE_CODE = 65535, PURE_MODULE_STACK = 64, + PURE_MODULE_NAME = 32, PURE_MODULE_STRINGS = 32768, PURE_MODULE_HEADER = 40, + PURE_MODULE_FIELDS = 64, PURE_MODULE_COLLECTIONS = 1, PURE_MODULE_CONSTANTS = 4096, + PURE_MODULE_CONSTANT_BYTES = 1 << 20, PURE_MODULE_RECORDS = 8, PURE_MODULE_RECORD_FIELDS = 64, + PURE_RECORD_FIELDS = 16, PURE_MODULE_JOINS = 8, PURE_APP_PENDING = 256 +}; + +typedef enum { + PURE_OP_CONST, PURE_OP_LOAD, PURE_OP_DT, PURE_OP_ADD, PURE_OP_SUB, PURE_OP_MUL, + PURE_OP_DIV, PURE_OP_NEG, PURE_OP_LT, PURE_OP_LE, PURE_OP_GT, PURE_OP_GE, + PURE_OP_EQ, PURE_OP_NE, PURE_OP_AND, PURE_OP_OR, PURE_OP_NOT, PURE_OP_MIN, + PURE_OP_MAX, PURE_OP_CLAMP, PURE_OP_STORE, PURE_OP_JZ, PURE_OP_JMP, + PURE_OP_FIELD, PURE_OP_DOC, PURE_OP_DOC_SIZE, PURE_OP_LEN, PURE_OP_REPLACE, + PURE_OP_REPLY, PURE_OP_REQUIRE, + PURE_OP_TEXT_CONST, PURE_OP_TEXT_LOAD, PURE_OP_TEXT_INPUT, PURE_OP_NUM_TEXT, + PURE_OP_CONCAT, PURE_OP_CHARS, PURE_OP_LINES, PURE_OP_TEXT_BYTES, + PURE_OP_CUR_LOAD, PURE_OP_CUR_START, PURE_OP_CUR_END, PURE_OP_CUR_MOVE, + PURE_OP_CUR_STORE, PURE_OP_TEXT_INSERT, PURE_OP_TEXT_REMOVE, PURE_OP_TEXT_LINE, + PURE_OP_TEXT_SET, PURE_OP_TEXT_FROM, PURE_OP_BYTES_FROM, PURE_OP_CUR_COUNT, + PURE_OP_LIST_COUNT, PURE_OP_LIST_APPEND, PURE_OP_LIST_INSERT, PURE_OP_LIST_REMOVE, + PURE_OP_ITEM_GET, PURE_OP_ITEM_SET, PURE_OP_INDEX_LOAD, PURE_OP_INDEX_STORE, + PURE_OP_INDEX_MOVE, PURE_OP_INDEX_FIRST, PURE_OP_INDEX_LAST, PURE_OP_INDEX_NUM, PURE_OP_ROW, + PURE_OP_PICK, PURE_OP_SLIDE, PURE_OP_FOLD, PURE_OP_TEXT_COMPARE, PURE_OP_LIST_FIND, PURE_OP_LIST_HAS, + PURE_OP_NOW, PURE_OP_FLOOR, PURE_OP_MOD, PURE_OP_SEND, + PURE_OP_COUNT +} PureOpcode; +/* FOLD's argument: the kind (bits 0-7), the list state (8-15) and the length + * of the block that follows (16-31), run once per item as a column of that + * list. count counts the items whose block leaves a non-zero number; sum, + * max and min take a value and an include flag from their block; first gives + * the index of the first item whose block leaves a non-zero number. */ +typedef enum { PURE_FOLD_COUNT, PURE_FOLD_SUM, PURE_FOLD_MAX, PURE_FOLD_MIN, PURE_FOLD_FIRST } PureFold; +/* TEXT_COMPARE's argument: the relation, in the order of the number + * comparisons (LT LE GT GE EQ NE); texts order by their bytes. */ +typedef enum { PURE_LESS, PURE_LESS_EQUAL, PURE_GREATER, PURE_GREATER_EQUAL, PURE_EQUAL, PURE_NOT_EQUAL } PureRelation; + +/* A timer ticks every `period` milliseconds while a host runs the app. An + * output is a message the app sends (SEND, in a handler; it has fields as a + * message has and no handler): what receives it is the host's wiring. */ +typedef enum { PURE_KIND_KEY = 1, PURE_KIND_MESSAGE = 2, PURE_KIND_TEXT = 3, PURE_KIND_TIMER = 4, + PURE_KIND_OUTPUT = 5 } PureInputKind; +enum { PURE_TIMER_LONGEST = 86400000, PURE_SOURCE = 64 }; +/* A message field: a u32le or a binary64 (fixed, at its offset in the + * payload's prefix), or bytes or text of any length. The prefix also holds + * each variable field's u32le length but the last one's (its offset is + * PURE_FIELD_REST), which takes what follows the others; the variable + * fields' bytes follow the prefix in declaration order. A text is UTF-8 and + * a number finite, or the message fails. */ +typedef enum { PURE_FIELD_U32 = 1, PURE_FIELD_BYTES = 2, PURE_FIELD_NUMBER = 3, PURE_FIELD_TEXT = 4 } PureFieldType; +enum { PURE_FIELD_REST = UINT32_MAX }; +typedef enum { + PURE_STATE_NUM = 0, PURE_STATE_TEXT = 1, PURE_STATE_CURSOR = 2, PURE_STATE_LIST = 3, PURE_STATE_INDEX = 4 +} PureStateType; +/* A cursor that stays before text inserted at it (a mark); other cursors go + * after it (a caret). */ +enum { PURE_CURSOR_STAYS = 1 }; +/* A table view has one row per item of a list, or per item its filter + * keeps; its columns are number or text views evaluated for one row at a + * time. */ +typedef enum { PURE_VIEW_NUM = 0, PURE_VIEW_TEXT = 1, PURE_VIEW_TABLE = 2 } PureViewType; +typedef enum { PURE_UNIT_CHAR = 0, PURE_UNIT_LINE = 1 } PureUnit; +typedef enum { PURE_EDGE_DOWN, PURE_EDGE_UP } PureEdge; +/* EXISTS: an item with that key is already in the keyed list (the event + * reports PURE_CONFLICT). */ +typedef enum { + PURE_FAULT_NONE, PURE_FAULT_DIVIDE, PURE_FAULT_NON_FINITE, PURE_FAULT_CLAMP, + PURE_FAULT_LIMIT, PURE_FAULT_NOT_FOUND, PURE_FAULT_ENCODING, PURE_FAULT_EXISTS +} PureFault; + +typedef struct { double value; uint32_t arg; uint8_t op; } PureOp; +typedef struct { uint32_t start, length; } PureBlock; +typedef struct { uint32_t event; PureBlock code; } PureHandler; +typedef struct { char name[PURE_MODULE_NAME]; uint8_t type; uint32_t offset; } PureField; +typedef struct { + uint8_t kind; + bool keyed; + uint16_t field_start, field_count; + uint32_t prefix; /* fixed payload bytes before the variable fields */ + uint32_t period; /* a timer's, in milliseconds */ +} PureInputShape; + +/* A record type: fields numbered from first, each a number or text. */ +typedef struct { uint16_t first, count; } PureRecord; + +typedef struct { + uint16_t states, inputs, views, handlers, fields, collections, constants, records, record_fields; + uint32_t code_count; + char state_names[PURE_MODULE_STATES][PURE_MODULE_NAME]; + char input_names[PURE_MODULE_INPUTS][PURE_MODULE_NAME]; + char view_names[PURE_MODULE_VIEWS][PURE_MODULE_NAME]; + char collection_names[PURE_MODULE_COLLECTIONS][PURE_MODULE_NAME]; + char input_keys[PURE_MODULE_INPUTS][PURE_MODULE_NAME]; + uint8_t state_type[PURE_MODULE_STATES]; + uint16_t state_ref[PURE_MODULE_STATES]; /* text: constant; cursor: text; list: record; index: list */ + uint8_t state_flags[PURE_MODULE_STATES]; /* cursor: PURE_CURSOR_STAYS */ + /* A keyed list: its key field + 1 (0: none). Its items stay in key order + * (numbers by value, texts by bytes), each key once, and keys do not change. */ + uint8_t state_key[PURE_MODULE_STATES]; + PureInputShape input[PURE_MODULE_INPUTS]; + PureField field[PURE_MODULE_FIELDS]; + double initial[PURE_MODULE_STATES]; + uint8_t view_type[PURE_MODULE_VIEWS]; + uint16_t view_table[PURE_MODULE_VIEWS]; /* a column's table view + 1, or 0 */ + uint16_t view_list[PURE_MODULE_VIEWS]; /* the list of a table or column */ + PureBlock view_code[PURE_MODULE_VIEWS]; + PureHandler handler[PURE_MODULE_HANDLERS]; + /* The code and the text constants, sized to the module in one block + * (storage) that load takes and pure_module_release gives back. */ + PureOp *code; + uint32_t *constant_offset, *constant_length; + unsigned char *constant_data; + void *storage; + char record_names[PURE_MODULE_RECORDS][PURE_MODULE_NAME]; + PureRecord record[PURE_MODULE_RECORDS]; + char record_field_names[PURE_MODULE_RECORD_FIELDS][PURE_MODULE_NAME]; + uint8_t record_field_text[PURE_MODULE_RECORD_FIELDS]; + double record_field_default[PURE_MODULE_RECORD_FIELDS]; /* number, or text constant index */ +} PureModule; + +/* Decode and verify into out, which must be zeroed (a static one is) or hold + * a loaded module: what it held is given back first. On failure out holds + * nothing and error names the rule. */ +PureResult pure_module_load(const void *bytes, size_t size, PureModule *out, + char *error, size_t error_size); +/* Give back a loaded module's code and constants (no app may still run it). */ +void pure_module_release(PureModule *); +/* Read a module file whole and load it (NOT_FOUND when it cannot be read). */ +PureResult pure_module_read(const char *path, PureModule *out, char *error, size_t error_size); +/* Index of a name, or -1. Views are those outside tables. */ +int pure_module_state(const PureModule *, const char *name); +int pure_module_input(const PureModule *, const char *name); +int pure_module_view(const PureModule *, const char *name); +int pure_module_column(const PureModule *, uint32_t table, const char *name); +/* The index state a number view shows as its position (a view that is just + * `at.position`), or -1: what a host moves with pure_app_point. */ +int pure_module_position(const PureModule *, uint32_t view); +/* The text state a text view shows as it is (a view that is just the + * state), or -1: a host may lend its stored extents instead of copying. */ +int pure_module_text_state(const PureModule *, uint32_t view); + +/* Keyed collections live in a data host (for example the tree store). One + * event opens at most one document, the message's key, at its current + * version; edits stay in that working version until the event commits it. + * A version's bytes field is the value's size. */ +typedef struct { + void *context; + PureResult (*open)(void *context, uint32_t collection, const char *key, + void **document, int64_t *revision); + PureVersion (*version)(void *context, void *document); + PureResult (*read)(void *context, void *document, PureVersion, size_t offset, size_t length, + bool (*visit)(void *, PureSpan), void *visit_context); + PureResult (*splice)(void *context, void *document, size_t offset, size_t remove, PureSpan insert); + PureResult (*commit)(void *context, void *document, int64_t *revision); + void (*close)(void *context, void *document); + /* Optional: the host keeps the app's states. state opens a runtime-owned + * value by name as a draft (a missing one is empty at revision 0); save + * publishes the given documents and state drafts together, all or + * nothing, and names each document's new revision. */ + PureResult (*state)(void *context, const char *name, PureDraft **out); + PureResult (*save)(void *context, void *const *documents, size_t document_count, PureDraft *const *states, + size_t count, int64_t *revisions); +} PureData; + +/* A byte or text value inside one event or view: a range of the payload, a + * document or text version, a module constant, a copy or the event's arena. */ +typedef struct { uint8_t source; uint16_t owner; PureVersion version; uint64_t offset, length; } PureBytes; +typedef struct { unsigned char *bytes; size_t used, capacity; } PureArena; + +/* One app instance: its states and at most one open event. Text states are + * runtime-owned drafts; cursors are positions the runtime keeps on character + * boundaries; lists are runtime-owned and an index always names an item + * (or 0 in an empty list). Every event is atomic: on failure nothing changes. + * + * Saving. When the data host keeps states the app starts from the stored + * ones. A save publishes, in one commit, the documents edited by messages and + * the states changed since the last save; if it fails, every event since the + * last save is undone. By default each key, text and message event saves + * before it returns (a frame waits for the next one). Deferred, events apply + * at once and a host saves a burst with one commit: an event is durable once + * a later save succeeds, and each key's document joins a burst once (a second + * message for it, or one past PURE_APP_PENDING keys, saves the burst first). */ +typedef struct { + const PureModule *module; + PureData data; + double values[PURE_MODULE_STATES]; /* numbers, cursor byte positions and list indices */ + PureDraft *texts[PURE_MODULE_STATES]; + PureList *lists[PURE_MODULE_STATES]; + PureUndo undo; /* list changes of the current event */ + PureDraft *list_drafts[PURE_MODULE_STATES]; /* stored lists, when kept */ + PureDraft *stored_values; /* numbers, cursors and indices, when kept */ + PureVersion saved[PURE_MODULE_STATES]; /* text and list versions at the last save */ + /* Events since the last save: what a failed save returns to, and the + * documents messages opened (held until the save). */ + bool deferred, open; + double durable_values[PURE_MODULE_STATES]; + PureVersion durable_texts[PURE_MODULE_STATES], durable_lists[PURE_MODULE_STATES]; + uint64_t durable_ticks, durable_events; + struct PurePending { void *document; char key[65]; bool edited; } *pending; /* grows to PURE_APP_PENDING */ + unsigned pending_count, pending_capacity; + PureFault fault; + uint32_t fault_pc; + uint64_t ticks, events; + /* The time events see as now, in milliseconds since 1970 (UTC). The host + * sets it, so a script can replay any time; views never see it. */ + double now; + uint64_t changes; /* counts every change of states, never back: what a filter's rows are fresh for */ + struct PureRows { uint32_t table; uint64_t changes; size_t *items, count, capacity; } *rows; /* a filtered table's items */ + unsigned row_tables; + /* Current message or text event, kept until pure_app_finish. */ + void *document; + PureSpan payload; + bool has_reply; + PureBytes reply; + unsigned char *copied; + int64_t revision; + PureArena arena, view_arena; + /* Links (runtime-owned, saved with the states): messages sent on outputs + * and not trimmed, in the order sent (the first sent_durable are saved; + * a trimmed one goes at the next save), each output's last number (and + * as saved), those of outputs a module dropped, and the last number + * delivered from each source (and as saved). */ + struct PureSent { uint32_t output; uint64_t seq; unsigned char *payload; size_t size; bool trimmed; } *sent; + size_t sent_count, sent_capacity, sent_durable; + uint64_t last_sent[PURE_MODULE_INPUTS], saved_sent[PURE_MODULE_INPUTS]; + struct PureCarried { char name[PURE_MODULE_NAME]; uint64_t last; } *carried; + unsigned carried_count; + struct PureHeard { char source[PURE_SOURCE + 1]; uint64_t seq, saved; } *heard; + unsigned heard_count, heard_capacity; + PureDraft *stored_links; + bool links_changed; /* since the last save */ + bool duplicate; /* the last delivery was heard before and not applied */ + struct { const char *source; uint64_t seq; bool pending; } delivering; + struct PureSnapshot *snapshot; /* the newest snapshot made, or NULL */ + PureRetired retired; /* list items let go since it was made */ + bool reading; /* a reader: its states are a snapshot's */ +} PureApp; + +PureResult pure_app_start(PureApp *, const PureModule *, PureData data); +/* Publish every change since the last save in one commit, or undo them all. */ +PureResult pure_app_save(PureApp *); +/* Defer saving to pure_app_save (false: save after each event). Needs a data + * host that saves documents together (PureData.save). */ +bool pure_app_defer(PureApp *, bool deferred); +/* Leaves unsaved frame changes unsaved: call pure_app_save first to keep them. */ +void pure_app_stop(PureApp *); +PureResult pure_app_frame(PureApp *); +/* A timer input's tick: an event saved like a key's. */ +PureResult pure_app_tick(PureApp *, uint32_t input); +PureResult pure_app_input(PureApp *, uint32_t input, PureEdge); +/* Point an index state at the item at position (0 first), as a user pointing + * at a row: an event with no handler, saved like a key event. NOT_FOUND past + * the last item, INVALID for a state that is not an index. */ +PureResult pure_app_point(PureApp *, uint32_t state, size_t position); +/* Deliver committed text (for example from an input method). Not UTF-8: invalid. */ +PureResult pure_app_text(PureApp *, uint32_t input, PureSpan text); +/* Deliver a message. base is the revision the sender edited, or -1 for none; + * a handler that edits the document needs one. payload stays alive until + * pure_app_finish. On return app->revision is the document's revision + * (deferred: the revision the next save gives it). */ +PureResult pure_app_message(PureApp *, uint32_t input, const char *key, int64_t base, PureSpan payload); +/* The same, with the variable fields where the host found them: field f + * starts at placed[2f] in payload and has placed[2f + 1] bytes (entries of + * fixed fields are not read, nor are the variable fields' lengths in the + * prefix). A host that parsed a request in place passes the fields' bytes + * where they are. */ +PureResult pure_app_message_fields(PureApp *, uint32_t input, const char *key, int64_t base, PureSpan payload, + const uint32_t *placed); +/* Visit the reply of the last message in order, without copying, until the + * next event (deferred: copy it before a later save too). */ +PureResult pure_app_reply(const PureApp *, bool (*visit)(void *, PureSpan), void *context); +void pure_app_finish(PureApp *); +/* A number view; false for a text view or a failed evaluation. */ +bool pure_app_view(PureApp *, uint32_t view, double *out); +/* A text view, lent in order until the next view evaluation. */ +PureResult pure_app_view_text(PureApp *, uint32_t view, bool (*visit)(void *, PureSpan), void *context); +/* The rows of a table view (its list's items, or those its filter keeps), + * or 0 for another view. A host evaluates only the cells it shows. */ +size_t pure_app_rows(PureApp *, uint32_t table); +/* The item a row of a table shows, and the row that shows an item (false: + * its filter leaves the item out). Without a filter both are the identity. */ +bool pure_app_row_item(PureApp *, uint32_t table, size_t row, size_t *item); +bool pure_app_item_row(PureApp *, uint32_t table, size_t item, size_t *row); +bool pure_app_cell(PureApp *, uint32_t column, size_t row, double *out); +PureResult pure_app_cell_text(PureApp *, uint32_t column, size_t row, bool (*visit)(void *, PureSpan), void *context); +/* Switch to another verified module. States keep their value by name (text + * states keep their content, cursors their position in that text); new states + * start at their initial value. The module must outlive the app. */ +PureResult pure_app_replace(PureApp *, const PureModule *); + +/* Connections between apps (docs/research/078-app-connections.md). A host + * sees only saved sends, delivers each at least once, in order per output, + * and trims what its receiver has saved; a receiver applies each once. */ +typedef bool (*PureSentVisit)(void *context, uint32_t output, uint64_t seq, PureSpan payload); +void pure_app_outbox(const PureApp *, PureSentVisit visit, void *context); +/* The same, newest first (a host looking only for what is new stops early). */ +void pure_app_outbox_newest(const PureApp *, PureSentVisit visit, void *context); +/* Forget output's saved sends numbered up to through (saved with the next save). */ +void pure_app_trim(PureApp *, uint32_t output, uint64_t through); +/* Deliver a message event from source (a sender's name, 1 to 64 printable + * ASCII bytes) numbered seq (from 1, growing per source). A number heard + * before is not applied again: it succeeds with app->duplicate set. The + * number counts as heard with the event's changes, and also when the app + * refuses the message (a fault: it was delivered, and refused); not when a + * save fails. placed is as for pure_app_message_fields, or NULL. */ +PureResult pure_app_deliver(PureApp *, uint32_t input, const char *source, uint64_t seq, PureSpan payload, + const uint32_t *placed); + +/* Readers on other threads. Right after a save an app can make a snapshot of + * its saved states, for evaluating views anywhere while it keeps applying + * events: numbers are copied; text states share their stored versions and + * lists their items (a list unchanged since the last snapshot shares that + * snapshot's copy), so the cost does not grow with the text. NULL when a + * change waits for a save, when a text state is not all stored (for example + * without a data host that keeps states), or when memory runs out. The caller + * gets a reference; the app keeps its own until the next snapshot. */ +typedef struct PureSnapshot PureSnapshot; +PureSnapshot *pure_app_snapshot(PureApp *); +void pure_snapshot_hold(PureSnapshot *); +/* On any thread; the last reference frees it. */ +void pure_snapshot_release(PureSnapshot *); +/* Show a snapshot's states in reader, a zeroed app or a reader before: its + * views read them while the snapshot is held, with the reader's own arena and + * row cache (one thread at a time). It applies no events. pure_app_stop frees + * the reader and leaves the snapshot alone. */ +void pure_app_read(PureApp *reader, const PureSnapshot *); +#endif diff --git a/core/result.h b/core/result.h new file mode 100644 index 0000000..8abb0f2 --- /dev/null +++ b/core/result.h @@ -0,0 +1,8 @@ +#ifndef PURE_RESULT_H +#define PURE_RESULT_H +typedef enum { + PURE_OK, PURE_NOT_FOUND, PURE_CONFLICT, PURE_BUSY, PURE_BUDGET, + PURE_LIMIT, PURE_STORE_ERROR, PURE_UNCERTAIN, + PURE_INVALID, PURE_DENIED, PURE_CLOSED, PURE_CANCELLED, PURE_ALLOCATION +} PureResult; +#endif diff --git a/core/runtime.c b/core/runtime.c new file mode 100644 index 0000000..4a857c7 --- /dev/null +++ b/core/runtime.c @@ -0,0 +1,394 @@ +#include "runtime.h" +#include + +#define NONE SIZE_MAX +enum { FREE, RECEIVING, READY, RUNNING, DONE }; +#ifdef PURE_RUNTIME_METRICS +#define METRIC(...) do { __VA_ARGS__; } while (0) +#else +#define METRIC(...) ((void)0) +#endif + +static PureWork *lookup(PureTicket ticket, bool require_held) { + PureRuntime *r = ticket.runtime; + if (!r || ticket.slot >= r->capacity || !ticket.generation) return NULL; + PureWork *w = &r->work[ticket.slot]; + if (w->state == FREE || w->generation != ticket.generation || + (require_held && !w->held)) return NULL; + return w; +} + +static bool valid_key(const char *key) { + if (!key || !*key) return false; + size_t n = 0; + for (; n <= 64 && key[n]; ++n) { + unsigned char c = (unsigned char)key[n]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '-' || c == '_')) return false; + } + return n <= 64; +} + +/* WRITE results are loans from the admitted input, not a second allocation. + * Integer offsets avoid relational comparisons between unrelated C pointers. */ +static bool borrowed_reply(PureSpan input, PureSpan reply) { + if (!reply.size) return true; + if (!input.data || !reply.data) return false; + uintptr_t base = (uintptr_t)input.data, start = (uintptr_t)reply.data; + return start >= base && start - base <= input.size && + reply.size <= input.size - (start - base); +} + +static void dispose(PureRuntime *r, PureResource *resource) { + PureResource old = *resource; + *resource = (PureResource){0}; + if (old.release) { + bool running = r->running; + r->running = true; + old.release(old.allocation, old.allocation_size); + r->running = running; + } +} + +static void payload_release(PureRuntime *r, PureWork *w); + +/* Iteratively retire a chain of forwarded values. An arbitrarily long chain + * must not turn completion into recursive stack growth. */ +static void retire(PureRuntime *r, PureWork *w) { + while (w && !w->held && !w->pins) { + PureTicket parent = w->parent; +#ifdef PURE_RUNTIME_METRICS + bool released_payload = w->charged || w->resource.release || parent.runtime; +#endif + dispose(r, &w->resource); + r->retained -= w->charged; + METRIC(if (released_payload) ++r->releases); + size_t slot = (size_t)(w - r->work); + *w = (PureWork){.next = r->free_head}; + r->free_head = slot; + --r->active; + if (!parent.runtime) break; + r = parent.runtime; + w = lookup(parent, false); + /* Pins can only be created and removed by the owner-thread core. */ + if (!w || !w->pins) break; + --w->pins; + } +} + +static void payload_release(PureRuntime *r, PureWork *w) { + PureTicket parent = w->parent; +#ifdef PURE_RUNTIME_METRICS + bool released_payload = w->charged || w->resource.release || parent.runtime; +#endif + dispose(r, &w->resource); + r->retained -= w->charged; + METRIC(if (released_payload) ++r->releases); + w->charged = 0; + w->parent = (PureTicket){0}; + if (parent.runtime) { + PureWork *source = lookup(parent, false); + if (source && source->pins) { --source->pins; retire(parent.runtime, source); } + } +} + +bool pure_runtime_init(PureRuntime *r, PureWork *work, size_t capacity, + const void *definition, PureHost host, size_t budget) { + if (!r || ((work == NULL) != (capacity == 0))) return false; + *r = (PureRuntime){.definition = definition, .host = host, .free_head = NONE, + .ready_head = NONE, .ready_tail = NONE, + .budget = budget, .accepting = true}; + return pure_runtime_bind(r, work, capacity); +} + +bool pure_runtime_bind(PureRuntime *r, PureWork *work, size_t capacity) { + if (!r || r->running || r->active || r->retained || + r->ready_head != NONE || r->ready_tail != NONE || + ((work == NULL) != (capacity == 0))) + return false; + r->work = work; + r->capacity = capacity; + r->free_head = capacity ? 0 : NONE; + for (size_t i = 0; i < capacity; ++i) + work[i] = (PureWork){.next = i + 1 < capacity ? i + 1 : NONE}; + return true; +} + +bool pure_runtime_restart(PureRuntime *r, const void *definition, PureHost host, size_t budget) { + if (!r || r->running || r->active) return false; + r->definition = definition; + r->host = host; + r->budget = budget; + r->accepting = true; + return true; +} + +void pure_runtime_stop(PureRuntime *r) { if (r && !r->running) r->accepting = false; } + +static PureResult admission(PureRuntime *r, PureInput input, const char *key, + int64_t expected, size_t bytes) { + if (!r || r->running || !valid_key(key) || + (input != PURE_INPUT_READ && input != PURE_INPUT_WRITE) || + (input == PURE_INPUT_READ ? bytes != 0 : expected < 0)) return PURE_INVALID; + if (!r->accepting) return PURE_CLOSED; + bool missing_effect = input == PURE_INPUT_READ ? !r->host.read : !r->host.write; +#ifdef PURE_ASYNC_EFFECTS + if (missing_effect && !r->async_effects) return PURE_DENIED; +#else + if (missing_effect) return PURE_DENIED; +#endif + if (r->free_head == NONE) return PURE_BUSY; + if (r->serial == UINT64_MAX) return PURE_LIMIT; + if (bytes > r->budget - r->retained) { METRIC(++r->budget_rejections); return PURE_BUDGET; } + return PURE_OK; +} + +static PureTicket occupy(PureRuntime *r, PureInput input, const char *key, + int64_t expected, size_t bytes, PureResource resource) { + size_t slot = r->free_head; + PureWork *w = &r->work[slot]; + r->free_head = w->next; + *w = (PureWork){.generation = ++r->serial, .next = NONE, .charged = bytes, + .state = RECEIVING, .held = true, .input = input, .expected = expected, + .resource = resource}; + memcpy(w->key, key, strlen(key) + 1); + ++r->active; + METRIC(++r->accepted); + r->retained += bytes; + METRIC(if (r->active > r->peak_active) r->peak_active = r->active); + METRIC(if (r->retained > r->peak_retained) r->peak_retained = r->retained); + return (PureTicket){r, slot, w->generation}; +} + +bool pure_runtime_ready(PureTicket ticket) { + PureWork *w = lookup(ticket, true); + if (!w || ticket.runtime->running || w->state != RECEIVING) return false; + PureRuntime *r = ticket.runtime; + w->state = READY; + if (r->ready_tail == NONE) r->ready_head = ticket.slot; + else r->work[r->ready_tail].next = ticket.slot; + r->ready_tail = ticket.slot; + return true; +} + +PureResult pure_runtime_accept(PureRuntime *r, PureInput input, const char *key, + int64_t expected, size_t size, PureTicket *out) { + if (!out) return PURE_INVALID; + PureResult status = admission(r, input, key, expected, size); + if (status != PURE_OK) return status; + PureResource resource = {0}; + if (size) { + if (!r->host.allocate) return PURE_ALLOCATION; + r->running = true; + bool ok = r->host.allocate(r->host.context, size, &resource); + r->running = false; + if (!ok || !resource.body.data || resource.body.size != size || !resource.release) { + dispose(r, &resource); + return PURE_ALLOCATION; + } + } + *out = occupy(r, input, key, expected, size, resource); + if (input == PURE_INPUT_READ) pure_runtime_ready(*out); + return PURE_OK; +} + +unsigned char *pure_runtime_ingress(PureTicket ticket, size_t *size) { + PureWork *w = lookup(ticket, true); + if (size) *size = 0; + if (!w || ticket.runtime->running || w->state != RECEIVING || + w->input != PURE_INPUT_WRITE || w->parent.runtime) return NULL; + if (size) *size = w->resource.body.size; + return (unsigned char *)w->resource.body.data; +} + +bool pure_runtime_step(PureRuntime *r) { + if (!r || r->running || r->ready_head == NONE) return false; +#ifdef PURE_ASYNC_EFFECTS + PureWork *head = &r->work[r->ready_head]; + if (head->input == PURE_INPUT_READ ? !r->host.read : !r->host.write) return false; +#endif + size_t slot = r->ready_head; + PureWork *w = &r->work[slot]; + r->ready_head = w->next; + if (r->ready_head == NONE) r->ready_tail = NONE; + w->next = NONE; + w->state = RUNNING; + r->running = true; + PureOutcome outcome = {0}; + if (w->input == PURE_INPUT_WRITE) { + PureSpan reply = {0}; + outcome.status = r->host.write(r->host.context, r->definition, w->key, + w->expected, w->resource.body, &reply, + &outcome.revision); + if (outcome.status == PURE_OK) { + if (borrowed_reply(w->resource.body, reply)) outcome.body = reply; + else outcome.status = PURE_STORE_ERROR; + } + METRIC(if (outcome.status == PURE_OK) ++r->commits; + else if (outcome.status == PURE_CONFLICT) ++r->conflicts); + } else { + outcome.status = r->host.read(r->host.context, r->definition, w->key, + r->budget - r->retained, &w->resource, &outcome.revision); + if (outcome.status == PURE_OK) { + if (w->resource.body.size > r->budget - r->retained) outcome.status = PURE_BUDGET; + else if (w->resource.body.size && !w->resource.release) + outcome.status = PURE_STORE_ERROR; + else { + w->charged = w->resource.body.size; + r->retained += w->charged; + METRIC(if (r->retained > r->peak_retained) r->peak_retained = r->retained); + outcome.body = w->resource.body; + METRIC(++r->gets); + } + } + METRIC(if (outcome.status == PURE_BUDGET) ++r->budget_rejections); + } + if (outcome.status != PURE_OK || !outcome.body.size) { + outcome.body = (PureSpan){0}; + payload_release(r, w); + } + METRIC(r->borrowed_output_bytes += outcome.body.size); + w->outcome = outcome; + w->state = DONE; + r->running = false; + return true; +} + +#ifdef PURE_ASYNC_EFFECTS +bool pure_runtime_enable_async(PureRuntime *r) { + if (!r || r->running || r->active) return false; + r->async_effects = true; + return true; +} + +PureTake pure_runtime_take(PureRuntime *r, PureEffect *out) { + if (!r || !out || !r->async_effects || r->running || r->ready_head == NONE) + return PURE_TAKE_EMPTY; + size_t slot = r->ready_head; + PureWork *w = &r->work[slot]; + r->ready_head = w->next; + if (r->ready_head == NONE) r->ready_tail = NONE; + w->next = NONE; + w->state = RUNNING; + *out = (PureEffect){.ticket = {r, slot, w->generation}, .input = w->input, + .key = w->key, .expected = w->expected, + .value = w->resource.body, + .available = r->budget - r->retained}; + return PURE_TAKE_EFFECT; +} + +bool pure_runtime_complete(PureTicket ticket, PureResult status, + PureResource *read_resource, PureSpan write_reply, + int64_t revision) { + PureWork *w = lookup(ticket, false); + PureRuntime *r = ticket.runtime; + if (!w || r->running || w->state != RUNNING || + (w->input == PURE_INPUT_WRITE && read_resource) || + (w->input == PURE_INPUT_READ && write_reply.size) || + (w->input == PURE_INPUT_WRITE && status == PURE_OK && + !borrowed_reply(w->resource.body, write_reply)) || + (read_resource && !read_resource->release && + (read_resource->body.size || read_resource->allocation))) return false; + r->running = true; + PureOutcome outcome = {.status = status, .revision = revision}; + if (w->input == PURE_INPUT_READ) { + PureResource acquired = {0}; + if (read_resource) { + acquired = *read_resource; + *read_resource = (PureResource){0}; + } + if (outcome.status == PURE_OK) { + if (acquired.body.size > r->budget - r->retained) + outcome.status = PURE_BUDGET; + else if (acquired.body.size && !acquired.release) + outcome.status = PURE_STORE_ERROR; + else { + w->resource = acquired; + acquired = (PureResource){0}; + w->charged = w->resource.body.size; + r->retained += w->charged; + METRIC(if (r->retained > r->peak_retained) r->peak_retained = r->retained); + outcome.body = w->resource.body; + METRIC(++r->gets); + } + } + dispose(r, &acquired); + METRIC(if (outcome.status == PURE_BUDGET) ++r->budget_rejections); + } else { + if (status == PURE_OK) outcome.body = write_reply; + METRIC(if (outcome.status == PURE_OK) ++r->commits; + else if (outcome.status == PURE_CONFLICT) ++r->conflicts); + } + if (outcome.status != PURE_OK || !outcome.body.size) { + outcome.body = (PureSpan){0}; + payload_release(r, w); + } + METRIC(r->borrowed_output_bytes += outcome.body.size); + w->outcome = outcome; + w->state = DONE; + r->running = false; + if (!w->held) retire(r, w); + return true; +} +#endif + +bool pure_runtime_result(PureTicket ticket, PureOutcome *out) { + PureWork *w = lookup(ticket, true); + if (!out || !w || ticket.runtime->running || w->state != DONE) return false; + *out = w->outcome; + return true; +} + +PureResult pure_runtime_forward(PureTicket source, PureRuntime *r, + const char *key, int64_t expected, PureTicket *out) { + PureWork *parent = lookup(source, true); + if (!out || !parent || source.runtime->running || parent->state != DONE || + parent->outcome.status != PURE_OK || (parent->outcome.body.size && !parent->outcome.body.data)) + return PURE_INVALID; + PureResult status = admission(r, PURE_INPUT_WRITE, key, expected, parent->outcome.body.size); + if (status != PURE_OK) return status; + if (parent->pins == SIZE_MAX) return PURE_LIMIT; + PureTicket ticket = occupy(r, PURE_INPUT_WRITE, key, expected, parent->outcome.body.size, + (PureResource){.body = parent->outcome.body}); + ++parent->pins; + r->work[ticket.slot].parent = source; + pure_runtime_ready(ticket); + *out = ticket; + return PURE_OK; +} + +static void unlink_ready(PureRuntime *r, size_t slot) { + size_t previous = NONE; + for (size_t at = r->ready_head; at != NONE; at = r->work[at].next) { + if (at == slot) { + if (previous == NONE) r->ready_head = r->work[at].next; + else r->work[previous].next = r->work[at].next; + if (r->ready_tail == at) r->ready_tail = previous; + r->work[at].next = NONE; + return; + } + previous = at; + } +} + +bool pure_runtime_cancel(PureTicket ticket) { + PureWork *w = lookup(ticket, true); + if (!w || ticket.runtime->running || (w->state != RECEIVING && w->state != READY)) return false; + if (w->state == READY) unlink_ready(ticket.runtime, ticket.slot); + payload_release(ticket.runtime, w); + w->outcome = (PureOutcome){.status = PURE_CANCELLED}; + w->state = DONE; + return true; +} + +bool pure_runtime_release(PureTicket ticket) { + PureWork *w = lookup(ticket, true); + if (!w || ticket.runtime->running) return false; + if (w->state == READY) unlink_ready(ticket.runtime, ticket.slot); + w->held = false; +#ifdef PURE_ASYNC_EFFECTS + if (w->state == RUNNING) return true; +#endif + retire(ticket.runtime, w); + return true; +} diff --git a/core/runtime.h b/core/runtime.h new file mode 100644 index 0000000..70f5d08 --- /dev/null +++ b/core/runtime.h @@ -0,0 +1,150 @@ +#ifndef PURE_RUNTIME_H +#define PURE_RUNTIME_H +#include "flow.h" +#include "result.h" +#include + +/* The optional metrics profile changes PureRuntime's layout. Give its init + * function a distinct link name so mixed-profile objects cannot link quietly. */ +#ifdef PURE_RUNTIME_METRICS +#define pure_runtime_init pure_runtime_init_metrics +#endif + +/* One owner thread controls admission, payload credits, completion and loans. + * An application definition is opaque to this core. Hosts interpret it and + * perform effects; no parser, storage, socket or OS type is linked here. */ +typedef struct PureRuntime PureRuntime; +typedef struct PureWork PureWork; +typedef struct { PureRuntime *runtime; size_t slot; uint64_t generation; } PureTicket; +typedef enum { PURE_INPUT_READ, PURE_INPUT_WRITE } PureInput; + +/* The host owns this resource until it successfully returns it to the core. + * release must end all access and free/unmap allocation exactly once. It runs + * while that domain is busy; calls back into the same domain are rejected. + * A resource the host sends in pieces (such as extents of a mapping) has no + * contiguous body: body.data is NULL and body.size is still what it holds and + * is charged. Its outcome has no body to read and cannot be forwarded. */ +typedef struct { + PureSpan body; + void *allocation; + size_t allocation_size; + void (*release)(void *allocation, size_t size); +} PureResource; +typedef struct { + void *context; + bool (*allocate)(void *, size_t, PureResource *); + PureResult (*read)(void *, const void *definition, const char *, + size_t available, PureResource *, int64_t *revision); + /* A successful reply may borrow only from input. The runtime retains its + * backing until all reply consumers finish. Effect errors publish no reply. */ + PureResult (*write)(void *, const void *definition, const char *, + int64_t expected, PureSpan input, PureSpan *reply, + int64_t *revision); +} PureHost; +typedef struct { PureResult status; int64_t revision; PureSpan body; } PureOutcome; + +/* Caller-provided work slots: no background thread or per-domain heap. These + * records are implementation storage, not an ABI or a wire representation. */ +/* Internal storage order packs the fixed key around small flags. This is not + * a public ABI; the layout is checked as part of the core cost gate. */ +struct PureWork { + PureTicket parent; + unsigned state; + PureInput input; + bool held; + char key[65]; + int64_t expected; + PureOutcome outcome; + PureResource resource; + uint64_t generation; + size_t next, charged, pins; +}; +struct PureRuntime { + const void *definition; + PureHost host; + PureWork *work; + size_t capacity, free_head, ready_head, ready_tail; + size_t budget, retained; +#ifdef PURE_RUNTIME_METRICS + size_t peak_retained; +#endif + size_t active; +#ifdef PURE_RUNTIME_METRICS + size_t peak_active; +#endif + uint64_t serial; +#ifdef PURE_RUNTIME_METRICS + uint64_t accepted, commits, gets, conflicts, budget_rejections; + uint64_t releases, borrowed_output_bytes; +#endif + bool accepting, running, async_effects; +}; + +/* Enforce repeated storage cost on 64-bit C hosts, including macOS where the + * Linux object-section checker is unavailable. */ +#if SIZE_MAX == UINT64_MAX && UINTPTR_MAX == UINT64_MAX +_Static_assert(sizeof(PureWork) == 216, "work-slot layout exceeded its reviewed size"); +#ifdef PURE_RUNTIME_METRICS +_Static_assert(sizeof(PureRuntime) == 192, "metrics domain layout changed"); +#else +_Static_assert(sizeof(PureRuntime) == 120, "domain layout changed"); +#endif +#endif + +/* Initialize fresh storage once. A NULL work pointer with capacity zero starts + * an idle domain; bind work before admission. Use restart, not init, to reuse + * a domain so its monotonic ticket sequence survives restart. definition is + * host-owned and remains alive until all work has drained. */ +bool pure_runtime_init(PureRuntime *, PureWork *, size_t capacity, + const void *definition, PureHost, size_t budget); +bool pure_runtime_restart(PureRuntime *, const void *definition, PureHost, size_t budget); +void pure_runtime_stop(PureRuntime *); /* stop admission, allow queued work to drain */ +/* Replace caller-owned work storage only after all tickets and loans retire. + * A zero-capacity binding releases idle slot storage without resetting budget, + * ticket generation or optional metrics. The caller retains the bound array's life. */ +bool pure_runtime_bind(PureRuntime *, PureWork *, size_t capacity); + +/* READ queues immediately. WRITE reserves its slot, budget and host allocation + * before returning. Fill ingress then publish once with ready. A partial input + * released without ready never executes. On error, out is unchanged. */ +PureResult pure_runtime_accept(PureRuntime *, PureInput, const char *key, + int64_t expected, size_t input_size, PureTicket *out); +unsigned char *pure_runtime_ingress(PureTicket, size_t *size); +bool pure_runtime_ready(PureTicket); +bool pure_runtime_step(PureRuntime *); /* execute at most one ready job */ +bool pure_runtime_result(PureTicket, PureOutcome *out); + +/* Pin a completed reply and queue it as another domain's WRITE input, with no + * allocation, serialization or payload copy. Source may be released as soon + * as this succeeds; the core retains its backing until the consumer ends. + * Each domain charges its own retained input bytes as admission credits. + * Credits across domains are NOT distinct physical memory and must not be summed. */ +PureResult pure_runtime_forward(PureTicket source, PureRuntime *destination, + const char *key, int64_t expected, PureTicket *out); +bool pure_runtime_cancel(PureTicket); /* only before execution; keeps a completion */ +bool pure_runtime_release(PureTicket); /* invalidates caller's ticket, preserves pins */ + +#ifdef PURE_ASYNC_EFFECTS +/* Optional owner-thread effect suspension. Enable before accepting work. The + * host may complete effects in any order, but must call complete on the owner + * thread after workers stop accessing its borrowed spans. An in-flight effect + * keeps its input alive even if the caller releases the ticket. Effects + * borrowed by take stay valid until completion. */ +typedef enum { PURE_TAKE_EMPTY, PURE_TAKE_EFFECT } PureTake; +typedef struct { + PureTicket ticket; + PureInput input; + const char *key; + int64_t expected; + PureSpan value; /* raw ingress for WRITE; empty for READ */ + size_t available; /* remaining payload credits for READ */ +} PureEffect; +bool pure_runtime_enable_async(PureRuntime *); +PureTake pure_runtime_take(PureRuntime *, PureEffect *); +/* A valid completion consumes read_resource (even on failure); a rejected call + * leaves it untouched. write_reply borrows from the retained ingress. */ +bool pure_runtime_complete(PureTicket, PureResult status, + PureResource *read_resource, PureSpan write_reply, + int64_t revision); +#endif +#endif diff --git a/core/state.h b/core/state.h new file mode 100644 index 0000000..2bf78b0 --- /dev/null +++ b/core/state.h @@ -0,0 +1,84 @@ +#ifndef PURE_STATE_H +#define PURE_STATE_H +#include "flow.h" +#include "result.h" +#include + +/* Runtime-owned working versions of stored values (selected tree store). + * A draft starts at the committed version of a key and changes only in + * memory: an edit copies the leaf and path it touches and shares every other + * extent with the stored version. An app therefore keeps no copy of the value + * and never compares a copy with storage to find what it changed. Commit + * appends only the draft's new extents. One owner thread per draft. */ +typedef struct PureDraft PureDraft; + +/* A working value with no stored base, for runtime-owned state such as an + * app's text. It edits and reads like any draft but cannot be committed. */ +PureDraft *pure_draft_new(void); +/* A missing key opens as an empty draft at revision 0; commit creates it. */ +PureResult pure_draft_open(const char *path, const char *key, PureDraft **out); +/* A runtime-owned value of an app, such as a list or text state, stored + * under a name that is never a document key (hosts listing documents do not + * see it). name is a valid key of at most 63 bytes. */ +PureResult pure_state_open(const char *path, const char *name, PureDraft **out); +/* A store kept open by one owner thread. Drafts opened through it share one + * mapping of the newest version, and opening one checks only whether the file + * changed; commits of its drafts map the new version once. It must outlive + * its drafts. */ +typedef struct PureStore PureStore; +PureResult pure_store_open(const char *path, PureStore **out); +void pure_store_close(PureStore *); +PureResult pure_store_draft(PureStore *, const char *key, PureDraft **out); +PureResult pure_store_state(PureStore *, const char *name, PureDraft **out); +/* Stored extents can outlive their draft: pure_draft_mapping takes a + * reference to the mapping a draft opened through a handle reads its stored + * nodes from (NULL for other drafts). Spans that mapping holds stay readable + * until the reference is released, after the draft closes and after the + * store moves on. Spans of a draft's unsaved edits are not in it. Take it on + * the handle's owner thread (a frozen draft's on any); the reference may be + * released on any thread. */ +typedef struct PureMapping PureMapping; +PureMapping *pure_draft_mapping(const PureDraft *); +bool pure_mapping_holds(const PureMapping *, PureSpan); +void pure_mapping_release(PureMapping *); +/* Pinned, drafts open at the version found when pinning, without checking + * the file each time (a host pins for one burst of requests); commits made + * through the handle still move it on. */ +PureResult pure_store_pin(PureStore *, bool pinned); +/* Memory only. Fails without change on a bad range or when memory runs out. */ +PureResult pure_draft_splice(PureDraft *, size_t offset, size_t remove, PureSpan insert); +size_t pure_draft_size(const PureDraft *); +/* The committed revision this draft is based on; 0 if the key was absent. */ +int64_t pure_draft_revision(const PureDraft *); +/* Visit [offset, offset + length) in order as borrowed extents, without + * copying. Returning false stops early. Spans stay valid until the next + * splice, commit or close of this draft. */ +PureResult pure_draft_read(const PureDraft *, size_t offset, size_t length, + bool (*visit)(void *, PureSpan), void *context); +/* An immutable version of a draft's value. It stays readable while the draft + * is held (reclamation is suspended) and until the next commit or close. */ +typedef struct { uint64_t ref, bytes; } PureVersion; +PureVersion pure_draft_version(const PureDraft *); +PureResult pure_draft_read_version(const PureDraft *, PureVersion, size_t offset, size_t length, + bool (*visit)(void *, PureSpan), void *context); +/* A read-only copy of a draft opened through a store handle, at its current + * version, when that version is all stored (no edits since its last commit, + * as right after one): it shares the draft's mapping, and since reading + * changes nothing it may be read, lend its mapping and close on any thread. + * NULL otherwise, or when memory runs out. */ +PureDraft *pure_draft_freeze(const PureDraft *); +/* While held, edits keep every earlier version's nodes (one transaction). */ +void pure_draft_hold(PureDraft *, bool held); +/* Abandon edits made since version was taken; the draft was held since then. */ +void pure_draft_restore(PureDraft *, PureVersion); +/* Publish if the key is still at the draft's base revision. On success the + * draft continues from the new revision; on any failure it is unchanged. */ +PureResult pure_draft_commit(PureDraft *, int64_t *revision); +/* Publish drafts of one store in one commit: each key must still be at its + * draft's base revision (a conflict names the stored revision in revisions), + * and either every draft is published or none is. Keys must differ. */ +PureResult pure_draft_commit_all(PureDraft *const *drafts, size_t count, int64_t *revisions); +/* Commits made so far in this process: groups formed, and requests in them. */ +void pure_store_group_counts(unsigned long *groups, unsigned long *requests); +void pure_draft_close(PureDraft *); +#endif diff --git a/core/store.h b/core/store.h new file mode 100644 index 0000000..dbfa8f8 --- /dev/null +++ b/core/store.h @@ -0,0 +1,54 @@ +#ifndef PURE_STORE_H +#define PURE_STORE_H +#include "flow.h" +#include "result.h" +#include + +#define PURE_OBJECT_LIMIT ((2u * 6u + 32u) * 32768u + 8192u) +typedef struct { + void *mapping; + size_t mapped_size, object_size; + PureSpan body; + int64_t revision; +} PureStoredView; + +PureResult pure_store_init(const char *path); +PureResult pure_store_get(const char *path, const char *key, size_t available, + PureStoredView *out); +PureResult pure_store_get_range(const char *path, const char *key, size_t offset, + size_t length, size_t available, PureStoredView *out); +/* Inspect once: guards and mapping refer to the same immutable file version. + * minimum preserves bounds checks from eliminated intermediate views. + * whole selects the suffix starting at offset; otherwise length is exact. */ +PureResult pure_store_select(const char *path, const char *key, size_t offset, + size_t length, bool whole, size_t minimum, + size_t available, PureStoredView *out); +void pure_store_release(PureStoredView *); +PureResult pure_store_put(const char *path, const char *key, int64_t expected, + PureSpan body, int64_t *revision); +/* Atomic byte replacement: retain [0,offset), insert bytes, retain suffix after + * remove bytes. Check the expected revision and bounds under the same writer + * lock. expected=0 permits creating an empty base (offset=remove=0). + * Retained bytes use bounded scratch space or a selected platform file copy; + * never assembles a whole new object in application memory. The immutable + * file is still rewritten. max_size is the final-body policy from the + * prepared app and is checked with the current size under the writer lock. */ +PureResult pure_store_splice(const char *path, const char *key, int64_t expected, + size_t offset, size_t remove, PureSpan insert, + size_t max_size, int64_t *revision); + +/* Visit each stored key once; the order is store-defined. Returning false + * from visit stops early. key is valid only during the call. Hosts that + * browse documents use this instead of reading a store's file layout. */ +PureResult pure_store_keys(const char *path, bool (*visit)(void *, const char *key), + void *context); + +/* Fault hooks exist only in the separately built unit-test binary. */ +typedef enum { + PURE_AFTER_HEADER, PURE_BEFORE_FILE_SYNC, PURE_BEFORE_RENAME, PURE_BEFORE_DIR_SYNC, + PURE_COPY_RANGE_UNSUPPORTED_AFTER_PARTIAL +} PureStoreStage; +#ifdef PURE_STORE_TEST +int pure_store_test_fault(PureStoreStage stage); +#endif +#endif diff --git a/core/test-module.c b/core/test-module.c new file mode 100644 index 0000000..ccb3371 --- /dev/null +++ b/core/test-module.c @@ -0,0 +1,2853 @@ +#define _POSIX_C_SOURCE 200809L +#include "module.h" +#include "module-store.h" +#include "module-text.h" +#include "store.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #x); abort(); } } while (0) + +/* Store faults: a stage fails (or the process dies there when crash is set). */ +static int fail_stage = -1; +static int crash; +int pure_store_test_fault(PureStoreStage stage) { + if ((int)stage != fail_stage) return 0; + if (crash) _exit(77); + return 1; +} + +typedef struct { const char *name; double initial; } S; +typedef struct { const char *name; uint32_t type; } F; +/* kind 0 is a key input; otherwise PURE_KIND_MESSAGE with fields. */ +typedef struct { const char *name, *key; int kind; bool keyed; const F *fields; size_t nfields; } I; +#define KEY(name, key) {name, key, 0, false, NULL, 0} +/* A timer's period (milliseconds) rides in nfields. */ +#define TIMER(name, period) {name, NULL, PURE_KIND_TIMER, false, NULL, period} +typedef struct { const char *name; uint32_t start, length; } V; +typedef struct { uint32_t event, start, length; } H; +typedef struct { uint8_t op; uint32_t arg; double value; } O; +/* Text (type 1, ref = constant), cursor (type 2, ref = text state), list + * (type 3, ref = record) and index (type 4, ref = list state) states follow + * the number states; text views follow the number views. */ +typedef struct { const char *name; uint32_t type, ref; } T; /* type: the state type, its flags from bit 8 */ +/* A record field: a number and its default, or text (text = 1) whose + * default is the constant numbered initial. */ +typedef struct { const char *name; uint32_t text; double initial; } RF; +typedef struct { const char *name; const RF *fields; size_t nfields; } R; +/* A further view: a table (type 2, start = list state) or a column of the + * view numbered table - 1. */ +typedef struct { const char *name; uint32_t type, table, start, length; } W; +typedef struct { + const S *s; size_t ns; const I *in; size_t ni; const V *v; size_t nv; + const H *h; size_t nh; const O *o; size_t no; const char *const *c; size_t nc; + const T *t; size_t nt; const char *const *k; size_t nk; const V *tv; size_t ntv; + const R *r; size_t nr; const W *w; size_t nw; +} Spec; + +static unsigned char *put32(unsigned char *p, uint32_t v) { + for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(v >> (8 * i)); + return p + 4; +} + +static unsigned char *put_double(unsigned char *p, double value) { + uint64_t bits; + memcpy(&bits, &value, sizeof(bits)); + put32(p, (uint32_t)bits); + return put32(p + 4, (uint32_t)(bits >> 32)); +} + +/* Test-side encoder for format v5; the toolchain has its own. */ +static size_t build(unsigned char *out, Spec spec) { + unsigned char strings[4096], blob[4096]; + size_t used = 0, blob_used = 0; + const char *text[512]; + uint32_t offset[512]; + size_t names = 0; + for (size_t i = 0; i < spec.ns; ++i) text[names++] = spec.s[i].name; + for (size_t i = 0; i < spec.nt; ++i) text[names++] = spec.t[i].name; + for (size_t i = 0; i < spec.nc; ++i) text[names++] = spec.c[i]; + for (size_t i = 0; i < spec.ni; ++i) { + text[names++] = spec.in[i].name; + if (!spec.in[i].kind) text[names++] = spec.in[i].key; + for (size_t f = 0; spec.in[i].kind != PURE_KIND_TIMER && f < spec.in[i].nfields; ++f) + text[names++] = spec.in[i].fields[f].name; + } + for (size_t i = 0; i < spec.nv; ++i) text[names++] = spec.v[i].name; + for (size_t i = 0; i < spec.ntv; ++i) text[names++] = spec.tv[i].name; + for (size_t i = 0; i < spec.nw; ++i) text[names++] = spec.w[i].name; + size_t record_names = names, record_fields = 0; + for (size_t i = 0; i < spec.nr; ++i) { + text[names++] = spec.r[i].name; + for (size_t f = 0; f < spec.r[i].nfields; ++f) text[names++] = spec.r[i].fields[f].name; + record_fields += spec.r[i].nfields; + } + for (size_t i = 0; i < names; ++i) { + size_t length = strlen(text[i]); + offset[i] = (uint32_t)used; + strings[used++] = (unsigned char)length; + memcpy(strings + used, text[i], length); + used += length; + } + size_t fields = 0; + for (size_t i = 0; i < spec.ni; ++i) fields += spec.in[i].kind == PURE_KIND_TIMER ? 0 : spec.in[i].nfields; + for (size_t i = 0; i < spec.nk; ++i) { memcpy(blob + blob_used, spec.k[i], strlen(spec.k[i])); blob_used += strlen(spec.k[i]); } + unsigned char *p = out; + memset(p, 0, 40); + memcpy(p, "PUREMOD5", 8); + size_t states = spec.ns + spec.nt, views = spec.nv + spec.ntv + spec.nw; + p[8] = (unsigned char)states; p[9] = (unsigned char)(states >> 8); + p[10] = (unsigned char)spec.ni; p[12] = (unsigned char)views; p[14] = (unsigned char)spec.nh; + put32(p + 16, (uint32_t)spec.no); + put32(p + 20, (uint32_t)used); + p[24] = (unsigned char)fields; p[26] = (unsigned char)spec.nc; p[28] = (unsigned char)spec.nk; + p[30] = (unsigned char)spec.nr; p[36] = (unsigned char)record_fields; + put32(p + 32, (uint32_t)blob_used); + p += 40; + size_t n = 0; + for (size_t i = 0; i < spec.ns; ++i) { p = put32(p, offset[n++]); p = put32(p, 0); p = put_double(p, spec.s[i].initial); } + for (size_t i = 0; i < spec.nt; ++i) { + p = put32(p, offset[n++]); + p = put32(p, spec.t[i].type & 0xFF); + p = put32(p, spec.t[i].ref); + p = put32(p, spec.t[i].type >> 8); + } + for (size_t i = 0; i < spec.nc; ++i) { p = put32(p, offset[n++]); p = put32(p, 1); p = put32(p, 1); } + uint32_t field_offsets[64], field_start = 0; + size_t nf = 0; + for (size_t i = 0; i < spec.ni; ++i) { + const I *in = &spec.in[i]; + p = put32(p, offset[n++]); + *p++ = (unsigned char)(in->kind ? in->kind : 1); *p++ = in->keyed; *p++ = 0; *p++ = 0; + bool timer = in->kind == PURE_KIND_TIMER; + p = put32(p, timer ? (uint32_t)in->nfields : in->kind ? 0 : offset[n++]); + uint32_t first = in->kind == PURE_KIND_MESSAGE || in->kind == PURE_KIND_OUTPUT ? field_start : 0; /* only messages and outputs have fields */ + *p++ = (unsigned char)first; *p++ = (unsigned char)(first >> 8); + *p++ = (unsigned char)(timer ? 0 : in->nfields); *p++ = 0; + for (size_t f = 0; !timer && f < in->nfields; ++f) field_offsets[nf++] = offset[n++]; + field_start += timer ? 0 : (uint32_t)in->nfields; + } + nf = 0; + for (size_t i = 0; i < spec.ni; ++i) + for (size_t f = 0; spec.in[i].kind != PURE_KIND_TIMER && f < spec.in[i].nfields; ++f) { + p = put32(p, field_offsets[nf++]); + p = put32(p, spec.in[i].fields[f].type); + } + for (size_t i = 0, at = 0; i < spec.nk; ++i) { p = put32(p, (uint32_t)at); p = put32(p, (uint32_t)strlen(spec.k[i])); at += strlen(spec.k[i]); } + size_t record_field = record_names; + for (size_t i = 0, first = 0; i < spec.nr; ++i) { + p = put32(p, offset[record_field]); + record_field += 1 + spec.r[i].nfields; + *p++ = (unsigned char)first; *p++ = 0; *p++ = (unsigned char)spec.r[i].nfields; *p++ = 0; + first += spec.r[i].nfields; + } + for (size_t i = 0, n = record_names; i < spec.nr; ++i, ++n) + for (size_t f = 0; f < spec.r[i].nfields; ++f) { + const RF *field = &spec.r[i].fields[f]; + p = put32(p, offset[++n]); + p = put32(p, field->text); + if (field->text) { p = put32(p, (uint32_t)field->initial); p = put32(p, 0); } + else p = put_double(p, field->initial); + } + for (size_t i = 0; i < spec.nv; ++i) { p = put32(p, offset[n++]); p = put32(p, 0); p = put32(p, spec.v[i].start); p = put32(p, spec.v[i].length); } + for (size_t i = 0; i < spec.ntv; ++i) { p = put32(p, offset[n++]); p = put32(p, 1); p = put32(p, spec.tv[i].start); p = put32(p, spec.tv[i].length); } + for (size_t i = 0; i < spec.nw; ++i) { + const W *w = &spec.w[i]; + p = put32(p, offset[n++]); + *p++ = (unsigned char)w->type; *p++ = (unsigned char)(w->type >> 8); /* a filtered table: its list + 1 */ + *p++ = (unsigned char)w->table; *p++ = (unsigned char)(w->table >> 8); + p = put32(p, w->start); + p = put32(p, w->length); + } + for (size_t i = 0; i < spec.nh; ++i) { p = put32(p, spec.h[i].event); p = put32(p, spec.h[i].start); p = put32(p, spec.h[i].length); } + for (size_t i = 0; i < spec.no; ++i) { + *p++ = spec.o[i].op; *p++ = 0; *p++ = 0; *p++ = 0; + p = put32(p, spec.o[i].arg); + p = put_double(p, spec.o[i].value); + } + memcpy(p, strings, used); + memcpy(p + used, blob, blob_used); + return (size_t)(p + used + blob_used - out); +} + +static PureModule module_a, module_b; +static char error[128]; + +static PureResult load(Spec spec, PureModule *out) { + static unsigned char bytes[65536]; + size_t size = build(bytes, spec); + return pure_module_load(bytes, size, out, error, sizeof(error)); +} + +#define LEN(a) (sizeof(a) / sizeof(*(a))) +#define SPEC(s, in, v, h, o) ((Spec){s, LEN(s), in, LEN(in), v, LEN(v), h, LEN(h), o, LEN(o), NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}) + +enum { C = PURE_OP_CONST, L = PURE_OP_LOAD, DT = PURE_OP_DT, ADD = PURE_OP_ADD, + MUL = PURE_OP_MUL, DIV = PURE_OP_DIV, GT = PURE_OP_GT, MIN = PURE_OP_MIN, + MAX = PURE_OP_MAX, CL = PURE_OP_CLAMP, ST = PURE_OP_STORE, JZ = PURE_OP_JZ, + JMP = PURE_OP_JMP, NEG = PURE_OP_NEG }; + +static const S counter_states[] = {{"count", 0}, {"x", 3}}; +static const I counter_inputs[] = {KEY("tap", "Space"), KEY("move", "ArrowRight")}; +static const V counter_views[] = {{"label", 0, 1}, {"width", 1, 6}}; +static const O counter_code[] = { + {L, 0, 0}, /* 0 view label */ + {C, 0, 40}, {L, 0, 0}, {C, 0, 20}, {MIN, 0, 0}, {C, 0, 8}, /* 1 view width */ + {MUL, 0, 0}, + /* 7: fixed-up below: 40 + min(count, 20) * 8 needs an ADD; see width2 */ + {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, /* 7 tap.down */ + /* 11 move.down: if x > 5 { x = 0 } else { x = x + 1 } */ + {L, 1, 0}, {C, 0, 5}, {GT, 0, 0}, {JZ, 18, 0}, {C, 0, 0}, {ST, 1, 0}, {JMP, 22, 0}, + {L, 1, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 1, 0}, + /* 22 frame: x = x + dt */ + {L, 1, 0}, {DT, 0, 0}, {ADD, 0, 0}, {ST, 1, 0}, + /* 26 view width2 = 40 + min(count, 20) * 8 */ + {C, 0, 40}, {L, 0, 0}, {C, 0, 20}, {MIN, 0, 0}, {C, 0, 8}, {MUL, 0, 0}, {ADD, 0, 0}, +}; +static const H counter_handlers[] = {{1, 7, 4}, {3, 11, 11}, {0, 22, 4}}; + +static void behaviour(void) { + /* width at 1..7 leaves two values: rejected at load. */ + CHECK(load(SPEC(counter_states, counter_inputs, counter_views, counter_handlers, counter_code), &module_a) == PURE_INVALID); + CHECK(strstr(error, "view must leave one value of its type")); + static const V views[] = {{"label", 0, 1}, {"width", 26, 7}}; + CHECK(load(SPEC(counter_states, counter_inputs, views, counter_handlers, counter_code), &module_a) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + double value; + CHECK(pure_app_view(&app, 1, &value) && value == 40); + for (int i = 0; i < 25; ++i) CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); + CHECK(app.values[0] == 25 && pure_app_view(&app, 1, &value) && value == 200); + CHECK(pure_app_input(&app, 0, PURE_EDGE_UP) == PURE_OK && app.events == 26); + /* if / else with forward JZ and JMP */ + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 4); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 5); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 6); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 0); + CHECK(pure_app_frame(&app) == PURE_OK && app.values[1] == 1.0 / 60.0 && app.ticks == 1); + CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_INVALID); + CHECK(pure_module_input(&module_a, "move") == 1 && pure_module_view(&module_a, "width") == 1 && + pure_module_state(&module_a, "missing") == -1); + + /* Replacement keeps values by name. */ + static const S next_states[] = {{"fresh", 7}, {"count", 99}}; + static const I no_inputs[1]; + static const V no_views[1]; + static const H no_handlers[1]; + static const O no_code[1]; + CHECK(load((Spec){next_states, 2, no_inputs, 0, no_views, 0, no_handlers, 0, no_code, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, &module_b) == PURE_OK); + CHECK(pure_app_replace(&app, &module_b) == PURE_OK); + CHECK(app.module == &module_b && app.values[0] == 7 && app.values[1] == 25); + CHECK(pure_app_replace(&app, &module_a) == PURE_OK); + CHECK(app.values[0] == 25 && app.values[1] == 3); + pure_app_stop(&app); +} + +/* A fault leaves every state exactly as before the event. */ +static void faults(void) { + static const S states[] = {{"a", 1}, {"b", 2}}; + static const I inputs[] = {KEY("one", "KeyA"), KEY("two", "KeyB"), KEY("three", "Digit3")}; + static const O code[] = { + {C, 0, 9}, {ST, 0, 0}, {C, 0, 1}, {C, 0, 0}, {DIV, 0, 0}, {ST, 1, 0}, /* 0 divide */ + {C, 0, 9}, {ST, 0, 0}, {C, 0, 1e308}, {C, 0, 10}, {MUL, 0, 0}, {ST, 1, 0}, /* 6 overflow */ + {C, 0, 9}, {ST, 0, 0}, {C, 0, 1}, {C, 0, 3}, {C, 0, 2}, {CL, 0, 0}, {ST, 1, 0}, /* 12 clamp */ + {C, 0, 5}, {ST, 0, 0}, /* 19 ok */ + {C, 0, 0}, {NEG, 0, 0}, {C, 0, 0}, {MIN, 0, 0}, /* 21 view -0 */ + {C, 0, 0}, {C, 0, 0}, {NEG, 0, 0}, {MAX, 0, 0}, /* 25 view +0 */ + }; + static const H handlers[] = {{1, 0, 6}, {3, 6, 6}, {5, 12, 7}, {5, 19, 2}}; + static const V views[] = {{"low", 21, 4}, {"high", 25, 4}}; + CHECK(load(SPEC(states, inputs, views, handlers, code), &module_a) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + const PureFault expected[] = {PURE_FAULT_DIVIDE, PURE_FAULT_NON_FINITE, PURE_FAULT_CLAMP}; + for (uint32_t i = 0; i < 3; ++i) { + CHECK(pure_app_input(&app, i, PURE_EDGE_DOWN) == PURE_INVALID && app.fault == expected[i]); + CHECK(app.values[0] == 1 && app.values[1] == 2 && app.events == 0); + } + /* A later handler of the same event does not run after a fault. */ + CHECK(app.fault_pc == 17); + double value; + CHECK(pure_app_view(&app, 0, &value) && value == 0 && signbit(value)); + CHECK(pure_app_view(&app, 1, &value) && value == 0 && !signbit(value)); + pure_app_stop(&app); +} + +typedef struct { const char *reason; Spec spec; PureResult result; } Rejection; + +static void rejections(void) { + static const S s1[] = {{"a", 0}}; + static const S dup[] = {{"a", 0}, {"a", 1}}; + static const S bad_name[] = {{"1a", 0}}; + static const S long_name[] = {{"abcdefghijklmnopqrstuvwxyz012345", 0}}; + static const S non_finite[] = {{"a", INFINITY}}; + static const I i1[] = {KEY("k", "Space")}; + static const I bad_key[] = {KEY("k", "KeyAA")}; + static const I dup_key[] = {KEY("k", "Space"), KEY("j", "Space")}; + static const V clash[] = {{"a", 0, 1}}; + static const V v_store[] = {{"v", 0, 2}}; + static const V v_dt[] = {{"v", 2, 1}}; + static const V v_empty[] = {{"v", 0, 0}}; + static const H none[1]; + static const O load_a[] = {{L, 0, 0}}; + static const O store[] = {{C, 0, 1}, {ST, 0, 0}, {DT, 0, 0}}; + static const H h0[] = {{0, 0, 2}}; + static const H h_bad_event[] = {{3, 0, 2}}; + static const O back[] = {{C, 0, 1}, {JZ, 0, 0}}; + static const O beyond[] = {{C, 0, 1}, {JZ, 3, 0}}; + static const O under[] = {{ADD, 0, 0}}; + static const O join[] = {{C, 0, 1}, {JZ, 4, 0}, {C, 0, 1}, {C, 0, 1}, {ST, 0, 0}}; + static const O leftover[] = {{C, 0, 1}, {C, 0, 2}}; + static const O unknown_state[] = {{L, 1, 0}, {ST, 0, 0}}; + static const O nan_const[] = {{C, 0, NAN}, {ST, 0, 0}}; + static const O noncanon[] = {{C, 0, 1}, {NEG, 1, 0}, {ST, 0, 0}}; + static const O unreachable[] = {{JMP, 2, 0}, {C, 0, 1}, {C, 0, 1}, {ST, 0, 0}}; + static const O bad_op[] = {{99, 0, 0}, {ST, 0, 0}}; + static const H h_all[] = {{0, 0, 1}}; + static const H h_two[] = {{0, 0, 2}}; + static const H h_five[] = {{0, 0, 5}}; + static const H h_four[] = {{0, 0, 4}}; + O overflow[66]; + for (int i = 0; i < 65; ++i) overflow[i] = (O){C, 0, 1}; + overflow[65] = (O){ST, 0, 0}; + const Rejection cases[] = { + {"duplicate name", {dup, 2, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"state name", {bad_name, 1, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"state name", {long_name, 1, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"non-finite initial value", {non_finite, 1, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"input key", {s1, 1, bad_key, 1, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"duplicate key", {s1, 1, dup_key, 2, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"duplicate name", {s1, 1, i1, 0, clash, 1, none, 0, load_a, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"view with an effect", {s1, 1, i1, 0, v_store, 1, none, 0, store, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"view with an effect", {s1, 1, i1, 0, v_dt, 1, none, 0, store, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"empty view", {s1, 1, i1, 0, v_empty, 1, none, 0, store, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"handler event", {s1, 1, i1, 1, clash, 0, h_bad_event, 1, store, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"branch not forward", {s1, 1, i1, 0, clash, 0, h_two, 1, back, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"branch not forward", {s1, 1, i1, 0, clash, 0, h_two, 1, beyond, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"stack underflow", {s1, 1, i1, 0, clash, 0, h_all, 1, under, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"join with different stacks", {s1, 1, i1, 0, clash, 0, h_five, 1, join, 5, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"handler must leave no value", {s1, 1, i1, 0, clash, 0, h_two, 1, leftover, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"no such state", {s1, 1, i1, 0, clash, 0, h_two, 1, unknown_state, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"non-canonical operand", {s1, 1, i1, 0, clash, 0, h_two, 1, nan_const, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"non-canonical operand", {s1, 1, i1, 0, clash, 0, h0, 1, noncanon, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"unreachable instruction", {s1, 1, i1, 0, clash, 0, h_four, 1, unreachable, 4, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"unknown opcode", {s1, 1, i1, 0, clash, 0, h_two, 1, bad_op, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + {"stack overflow", {s1, 1, i1, 0, clash, 0, (H[]){{0, 0, 66}}, 1, overflow, 66, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, + }; + for (size_t i = 0; i < LEN(cases); ++i) { + PureResult result = load(cases[i].spec, &module_a); + if (result != cases[i].result || !strstr(error, cases[i].reason)) { + fprintf(stderr, "case %zu: expected '%s', got %d '%s'\n", i, cases[i].reason, (int)result, error); + abort(); + } + } + static S many[33]; + static char many_names[33][8]; + for (unsigned i = 0; i < 33; ++i) { snprintf(many_names[i], 8, "s%u", i % 100u); many[i] = (S){many_names[i], 0}; } + CHECK(load((Spec){many, 33, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, &module_a) == PURE_LIMIT); + static unsigned char bytes[65536]; + size_t size = build(bytes, (Spec){s1, 1, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}); + CHECK(pure_module_load(bytes, size - 1, &module_a, error, sizeof(error)) == PURE_INVALID); + CHECK(pure_module_load(bytes, size + 1, &module_a, error, sizeof(error)) == PURE_INVALID); + bytes[0] = 'X'; + CHECK(pure_module_load(bytes, size, &module_a, error, sizeof(error)) == PURE_INVALID); +} + +static uint32_t next_random(uint32_t *state) { + *state ^= *state << 13; + *state ^= *state >> 17; + *state ^= *state << 5; + return *state; +} + +/* Arbitrary damage is either rejected or yields a module that runs safely. */ +static void mutations(void) { + static const V views[] = {{"label", 0, 1}, {"width", 26, 7}}; + static unsigned char valid[65536], bytes[65536]; + size_t size = build(valid, SPEC(counter_states, counter_inputs, views, counter_handlers, counter_code)); + uint32_t state = 0x2545f491u; + unsigned accepted = 0; + for (unsigned round = 0; round < 20000; ++round) { + memcpy(bytes, valid, size); + unsigned flips = 1 + next_random(&state) % 4; + for (unsigned f = 0; f < flips; ++f) + bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); + if (pure_module_load(bytes, size, &module_a, error, sizeof(error)) != PURE_OK) continue; + ++accepted; + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + for (unsigned e = 0; e < 16; ++e) { + if (e % 3 == 0) pure_app_frame(&app); + else pure_app_input(&app, e % 4, e & 1 ? PURE_EDGE_UP : PURE_EDGE_DOWN); + for (uint32_t v = 0; v < module_a.views; ++v) { double value; pure_app_view(&app, v, &value); } + for (uint32_t s = 0; s < module_a.states; ++s) CHECK(isfinite(app.values[s])); + } + pure_app_stop(&app); + } + printf("{\"module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); +} + +static Spec editor_spec; + +/* An in-memory data host: every splice makes a new immutable version. */ +typedef struct { char key[65]; unsigned char *bytes; size_t size; int64_t revision; } Stored; +typedef struct { + Stored docs[8]; + size_t count, limit; + bool busy; + unsigned opened, closed; +} Memory; +typedef struct { + Memory *memory; + int index; /* -1 for an absent key */ + int64_t base; + unsigned char *versions[64]; + size_t sizes[64], count; +} Doc; + +static Stored *stored(Memory *m, const char *key) { + for (size_t i = 0; i < m->count; ++i) if (!strcmp(m->docs[i].key, key)) return &m->docs[i]; + return NULL; +} + +static PureResult fake_open(void *context, uint32_t collection, const char *key, void **out, int64_t *revision) { + Memory *m = context; + CHECK(collection == 0); + Doc *d = calloc(1, sizeof(*d)); + Stored *s = stored(m, key); + d->memory = m; + d->index = s ? (int)(s - m->docs) : -1; + if (!s) { + CHECK(m->count < 8); + d->index = (int)m->count; + snprintf(m->docs[m->count++].key, 65, "%s", key); + s = &m->docs[d->index]; + } + d->base = s->revision; + d->versions[0] = malloc(s->size + 1); + memcpy(d->versions[0], s->bytes ? s->bytes : (unsigned char *)"", s->size); + d->sizes[0] = s->size; + d->count = 1; + ++m->opened; + *out = d; + *revision = s->revision; + return PURE_OK; +} + +static PureVersion fake_version(void *context, void *document) { + (void)context; + Doc *d = document; + return (PureVersion){d->count - 1, d->sizes[d->count - 1]}; +} + +/* Two spans per read, so replies arrive in pieces. */ +static PureResult fake_read(void *context, void *document, PureVersion at, size_t offset, size_t length, + bool (*visit)(void *, PureSpan), void *visit_context) { + (void)context; + Doc *d = document; + CHECK(at.ref < d->count && at.bytes == d->sizes[at.ref] && offset + length <= at.bytes); + size_t half = length / 2; + if (half && !visit(visit_context, (PureSpan){d->versions[at.ref] + offset, half})) return PURE_OK; + if (length - half) visit(visit_context, (PureSpan){d->versions[at.ref] + offset + half, length - half}); + return PURE_OK; +} + +static PureResult fake_splice(void *context, void *document, size_t offset, size_t remove, PureSpan insert) { + Memory *m = context; + Doc *d = document; + size_t size = d->sizes[d->count - 1], next = size - remove + insert.size; + if (next > m->limit) return PURE_LIMIT; + CHECK(d->count < 64); + unsigned char *bytes = malloc(next + 1); + memcpy(bytes, d->versions[d->count - 1], offset); + if (insert.size) memcpy(bytes + offset, insert.data, insert.size); + memcpy(bytes + offset + insert.size, d->versions[d->count - 1] + offset + remove, size - offset - remove); + d->versions[d->count] = bytes; + d->sizes[d->count++] = next; + return PURE_OK; +} + +static PureResult fake_commit(void *context, void *document, int64_t *revision) { + Memory *m = context; + Doc *d = document; + Stored *s = &m->docs[d->index]; + *revision = s->revision; + if (m->busy) return PURE_BUSY; + if (s->revision != d->base) return PURE_CONFLICT; + free(s->bytes); + s->size = d->sizes[d->count - 1]; + s->bytes = malloc(s->size + 1); + memcpy(s->bytes, d->versions[d->count - 1], s->size); + *revision = ++s->revision; + /* Like a tree draft, a commit rebases: earlier versions are gone. */ + for (size_t i = 0; i + 1 < d->count; ++i) { free(d->versions[i]); d->versions[i] = NULL; d->sizes[i] = 0; } + d->versions[0] = d->versions[d->count - 1]; + d->sizes[0] = d->sizes[d->count - 1]; + d->count = 1; + d->base = s->revision; + return PURE_OK; +} + +static void fake_close(void *context, void *document) { + Memory *m = context; + Doc *d = document; + for (size_t i = 0; i < d->count; ++i) free(d->versions[i]); + free(d); + ++m->closed; +} + +static PureData fake_data(Memory *m) { + return (PureData){m, fake_open, fake_version, fake_read, fake_splice, fake_commit, fake_close, NULL, NULL}; +} + +typedef struct { unsigned char bytes[256]; size_t size, spans; } Collected; + +static bool count_bytes(void *context, PureSpan span) { + *(size_t *)context += span.size; + return true; +} + +static bool collect(void *context, PureSpan span) { + Collected *c = context; + CHECK(c->size + span.size <= sizeof(c->bytes)); + memcpy(c->bytes + c->size, span.data, span.size); + c->size += span.size; + ++c->spans; + return true; +} + +static bool replied(const PureApp *app, const char *text) { + Collected c = {{0}, 0, 0}; + CHECK(pure_app_reply(app, collect, &c) == PURE_OK); + return c.size == strlen(text) && !memcmp(c.bytes, text, c.size); +} + +static PureSpan edit_payload(unsigned char *buffer, uint32_t start, uint32_t remove, const char *insert) { + put32(buffer, start); + put32(buffer + 4, remove); + memcpy(buffer + 8, insert, strlen(insert)); + return (PureSpan){buffer, 8 + strlen(insert)}; +} + +enum { FIELD = PURE_OP_FIELD, DOC = PURE_OP_DOC, SIZE = PURE_OP_DOC_SIZE, LEN_ = PURE_OP_LEN, + REPL = PURE_OP_REPLACE, REPLY = PURE_OP_REPLY, REQ = PURE_OP_REQUIRE, LE = PURE_OP_LE }; + +/* The document editor as a module: show replies the document, edit splices + * start .. start + remove with insert, replies insert and counts edits. A + * keep message replies the document read before its own edit; guarded edits + * require the final size to stay within 8 bytes. */ +static void messages(void) { + static const F edit_fields[] = {{"start", 1}, {"remove", 1}, {"insert", 2}}; + static const I inputs[] = { + {"show", NULL, PURE_KIND_MESSAGE, true, NULL, 0}, + {"edit", NULL, PURE_KIND_MESSAGE, true, edit_fields, 3}, + {"keep", NULL, PURE_KIND_MESSAGE, true, edit_fields + 2, 1}, + {"guarded", NULL, PURE_KIND_MESSAGE, true, edit_fields, 3}, + KEY("tick", "Space"), + }; + static const S states[] = {{"edits", 0}}; + static const char *const collections[] = {"documents"}; + static const O code[] = { + {DOC, 0, 0}, {REPLY, 0, 0}, /* 0 show */ + {FIELD, 0, 0}, {FIELD, 0, 0}, {FIELD, 1, 0}, {ADD, 0, 0}, {FIELD, 2, 0}, /* 2 edit */ + {REPL, 0, 0}, {FIELD, 2, 0}, {REPLY, 0, 0}, {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, + {DOC, 0, 0}, {REPLY, 0, 0}, {C, 0, 0}, {C, 0, 0}, {FIELD, 0, 0}, {REPL, 0, 0}, /* 14 keep: prepend */ + {FIELD, 0, 0}, {FIELD, 0, 0}, {FIELD, 1, 0}, {ADD, 0, 0}, {FIELD, 2, 0}, /* 20 guarded */ + {REPL, 0, 0}, {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, + {SIZE, 0, 0}, {C, 0, 8}, {LE, 0, 0}, {REQ, 0, 0}, + }; + static const H handlers[] = {{1, 0, 2}, {3, 2, 12}, {5, 14, 6}, {7, 20, 14}}; + static const V views[] = {{"count", 0, 0}}; + Spec spec = {states, 1, inputs, 5, views, 0, handlers, 4, code, LEN(code), collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; + editor_spec = spec; + PureResult loaded = load(spec, &module_a); + if (loaded != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } + Memory memory = {.limit = 100}; + PureApp app; + CHECK(pure_app_start(&app, &module_a, fake_data(&memory)) == PURE_OK); + unsigned char buffer[64]; + + CHECK(pure_app_message(&app, 0, "doc", -1, (PureSpan){0}) == PURE_NOT_FOUND && app.revision == 0); + CHECK(pure_app_message(&app, 1, "doc", 0, edit_payload(buffer, 0, 0, "hello")) == PURE_OK); + CHECK(app.revision == 1 && replied(&app, "hello") && app.values[0] == 1); + CHECK(pure_app_message(&app, 0, "doc", -1, (PureSpan){0}) == PURE_OK && app.revision == 1 && replied(&app, "hello")); + /* stale base conflicts before bounds; bounds fail as limit; nothing changes */ + CHECK(pure_app_message(&app, 1, "doc", 0, edit_payload(buffer, 99, 0, "x")) == PURE_CONFLICT && app.revision == 1); + CHECK(pure_app_message(&app, 1, "doc", 1, edit_payload(buffer, 99, 0, "x")) == PURE_LIMIT && app.revision == 1); + CHECK(pure_app_message(&app, 1, "doc", 1, edit_payload(buffer, 2, 9, "x")) == PURE_LIMIT); + CHECK(pure_app_message(&app, 1, "doc", 1, (PureSpan){buffer, 7}) == PURE_LIMIT && app.revision == 0); + CHECK(pure_app_message(&app, 1, "doc", -1, edit_payload(buffer, 0, 0, "x")) == PURE_INVALID); + CHECK(pure_app_message(&app, 1, "bad/key", 1, edit_payload(buffer, 0, 0, "x")) == PURE_INVALID); + CHECK(pure_app_message(&app, 4, "doc", 1, (PureSpan){0}) == PURE_INVALID); + CHECK(memory.docs[0].revision == 1 && memory.docs[0].size == 5 && app.values[0] == 1); + /* A busy commit changes neither the document nor the states. */ + memory.busy = true; + CHECK(pure_app_message(&app, 1, "doc", 1, edit_payload(buffer, 5, 0, "!")) == PURE_BUSY); + memory.busy = false; + CHECK(memory.docs[0].size == 5 && app.values[0] == 1 && app.events == 2); + /* The size limit of the data host fails as limit. */ + memory.limit = 6; + CHECK(pure_app_message(&app, 1, "doc", 1, edit_payload(buffer, 5, 0, "!!")) == PURE_LIMIT); + memory.limit = 100; + /* A reply read before the event's own edit keeps its content after commit. */ + CHECK(pure_app_message(&app, 2, "doc", 1, (PureSpan){(const unsigned char *)">> ", 3}) == PURE_OK); + CHECK(app.revision == 2 && replied(&app, "hello")); + CHECK(pure_app_message(&app, 0, "doc", -1, (PureSpan){0}) == PURE_OK && replied(&app, ">> hello")); + /* require: an edit that would exceed 8 bytes changes nothing */ + CHECK(pure_app_message(&app, 3, "doc", 2, edit_payload(buffer, 8, 0, "!")) == PURE_LIMIT); + CHECK(memory.docs[0].size == 8 && app.values[0] == 1); + CHECK(pure_app_message(&app, 3, "doc", 2, edit_payload(buffer, 0, 3, "")) == PURE_OK && app.revision == 3); + CHECK(app.values[0] == 2); + /* Another key is its own document. */ + CHECK(pure_app_message(&app, 1, "other", 0, edit_payload(buffer, 0, 0, "abc")) == PURE_OK && app.revision == 1); + pure_app_stop(&app); + CHECK(memory.opened == memory.closed); + for (size_t i = 0; i < memory.count; ++i) free(memory.docs[i].bytes); + + /* Verifier rules for typed operations. */ + static const O field_in_key[] = {{FIELD, 0, 0}, {ST, 0, 0}}; + static const H key_handler[] = {{9, 0, 2}}; + Spec bad = spec; + bad.o = field_in_key; bad.no = 2; bad.h = key_handler; bad.nh = 1; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "message operation outside")); + static const O num_insert[] = {{C, 0, 0}, {C, 0, 0}, {C, 0, 1}, {REPL, 0, 0}}; + static const H edit_handler[] = {{3, 0, 4}}; + bad.o = num_insert; bad.no = 4; bad.h = edit_handler; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); + static const O bytes_state[] = {{FIELD, 2, 0}, {ST, 0, 0}}; + static const H edit_handler2[] = {{3, 0, 2}}; + bad.o = bytes_state; bad.no = 2; bad.h = edit_handler2; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); + static const O far_field[] = {{FIELD, 3, 0}, {REPLY, 0, 0}}; + bad.o = far_field; bad.h = edit_handler2; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "no such state, field")); + static const I unkeyed[] = {{"note", NULL, PURE_KIND_MESSAGE, false, NULL, 0}}; + static const O doc_read[] = {{DOC, 0, 0}, {REPLY, 0, 0}}; + static const H note_handler[] = {{1, 0, 2}}; + bad = (Spec){states, 1, unkeyed, 1, views, 0, note_handler, 1, doc_read, 2, collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "document operation outside")); + static const F unknown_type[] = {{"insert", 5}, {"start", 1}}; + static const I misplaced[] = {{"edit", NULL, PURE_KIND_MESSAGE, true, unknown_type, 2}}; + bad = (Spec){states, 1, misplaced, 1, views, 0, NULL, 0, NULL, 0, collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "message field")); + static const H even[] = {{2, 0, 0}}; + bad = (Spec){states, 1, inputs, 1, views, 0, even, 1, NULL, 0, collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "handler event")); + static const V field_view[] = {{"v", 0, 1}}; + static const O field_only[] = {{FIELD, 0, 0}}; + bad = (Spec){states, 1, inputs, 2, field_view, 1, NULL, 0, field_only, 1, collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view with an effect")); + /* A value may cross a join when every path brings the same stack. */ + static const O branch_value[] = {{C, 0, 1}, {C, 0, 1}, {JZ, 3, 0}, {ST, 0, 0}}; + static const O branch_mismatch[] = {{C, 0, 1}, {C, 0, 1}, {JZ, 4, 0}, {C, 0, 2}, {ST, 0, 0}}; + static const H frame_handler[] = {{0, 0, 4}}; + bad = (Spec){states, 1, inputs, 0, views, 0, frame_handler, 1, branch_value, 4, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_OK); + static const H frame_five[] = {{0, 0, 5}}; + bad.o = branch_mismatch; bad.no = 5; bad.h = frame_five; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "join with different stacks")); +} + +/* Damaged message modules either fail to load or run messages safely. */ +static void message_mutations(void) { + static unsigned char valid[65536], bytes[65536]; + size_t size = build(valid, editor_spec); + uint32_t state = 0x9e3779b9u; + unsigned accepted = 0; + for (unsigned round = 0; round < 20000; ++round) { + memcpy(bytes, valid, size); + unsigned flips = 1 + next_random(&state) % 3; + for (unsigned f = 0; f < flips; ++f) + bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); + if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; + ++accepted; + Memory memory = {.limit = 64}; + PureApp app; + CHECK(pure_app_start(&app, &module_b, fake_data(&memory)) == PURE_OK); + unsigned char payload[32]; + for (unsigned e = 0; e < 12; ++e) { + for (size_t k = 0; k < sizeof(payload); ++k) payload[k] = (unsigned char)next_random(&state); + put32(payload, next_random(&state) % 12); + put32(payload + 4, next_random(&state) % 4); + PureSpan span = {payload, next_random(&state) % sizeof(payload)}; + uint32_t input = next_random(&state) % 5; + if (module_b.input[input].kind == PURE_KIND_MESSAGE) { + int64_t base = (int64_t)(next_random(&state) % 4) - 1; + if (pure_app_message(&app, input, "doc", base, span) == PURE_OK) { + size_t reply = 0; + CHECK(pure_app_reply(&app, count_bytes, &reply) == PURE_OK && reply <= 256); + } + } else pure_app_input(&app, input, e & 1 ? PURE_EDGE_UP : PURE_EDGE_DOWN); + pure_app_frame(&app); + for (uint32_t s = 0; s < module_b.states; ++s) CHECK(isfinite(app.values[s])); + } + pure_app_stop(&app); + CHECK(memory.opened == memory.closed); + for (size_t i = 0; i < memory.count; ++i) free(memory.docs[i].bytes); + } + printf("{\"message_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); +} + +static Spec text_spec; + +typedef struct { char text[256]; size_t size; } Text; + +static bool text_span(void *context, PureSpan span) { + Text *t = context; + CHECK(t->size + span.size < sizeof(t->text)); + memcpy(t->text + t->size, span.data, span.size); + t->size += span.size; + t->text[t->size] = 0; + return true; +} + +static const char *view_text(PureApp *app, uint32_t view) { + static Text t; + t.size = 0; + t.text[0] = 0; + CHECK(pure_app_view_text(app, view, text_span, &t) == PURE_OK); + return t.text; +} + +static const char *state_text(const PureApp *app, uint32_t state) { + static Text t; + t.size = 0; + t.text[0] = 0; + CHECK(pure_draft_read(app->texts[state], 0, pure_draft_size(app->texts[state]), text_span, &t) == PURE_OK); + return t.text; +} + +static double number_view(PureApp *app, uint32_t view) { + double value; + CHECK(pure_app_view(app, view, &value)); + return value; +} + +enum { CUL = PURE_OP_CUR_LOAD, CS = PURE_OP_CUR_STORE, CM = PURE_OP_CUR_MOVE, TI = PURE_OP_TEXT_INPUT, + TINS = PURE_OP_TEXT_INSERT, TREM = PURE_OP_TEXT_REMOVE, TL = PURE_OP_TEXT_LOAD, TK = PURE_OP_TEXT_CONST, + CHR = PURE_OP_CHARS, NT = PURE_OP_NUM_TEXT, CAT = PURE_OP_CONCAT, TLN = PURE_OP_TEXT_LINE, + CC = PURE_OP_CUR_COUNT, CST = PURE_OP_CUR_START }; + +/* A small text editor module: states edits (num), note (text), at (cursor in + * note), title (text). The runtime keeps at on character boundaries and + * moves it with edits; a failed event restores text and cursor. */ +static void texts(void) { + static const S nums[] = {{"edits", 0}}; + static const T textual[] = {{"note", 1, 0}, {"at", 2, 1}, {"title", 1, 1}}; + static const char *const constants[] = {"\xea\xb0\x80" "a\nbc", "\xeb\xa9\x94\xeb\xaa\xa8", "XYZ", + " \xc2\xb7 ", "\xec\x9e\x90"}; + static const I inputs[] = { + {"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("left", "ArrowLeft"), KEY("right", "ArrowRight"), + KEY("up", "ArrowUp"), KEY("down", "ArrowDown"), KEY("back", "Backspace"), KEY("home", "Home"), KEY("bad", "KeyB"), + }; + static const O code[] = { + {CUL, 2, 0}, {TI, 0, 0}, {TINS, 1, 0}, {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, /* 0 typed */ + {CUL, 2, 0}, {C, 0, -1}, {CM, 0, 0}, {CS, 2, 0}, /* 7 left */ + {CUL, 2, 0}, {C, 0, 1}, {CM, 0, 0}, {CS, 2, 0}, /* 11 right */ + {CUL, 2, 0}, {C, 0, -1}, {CM, 1, 0}, {CS, 2, 0}, /* 15 up */ + {CUL, 2, 0}, {C, 0, 1}, {CM, 1, 0}, {CS, 2, 0}, /* 19 down */ + {CUL, 2, 0}, {C, 0, -1}, {CM, 0, 0}, {CUL, 2, 0}, {TREM, 1, 0}, /* 23 back */ + {CST, 1, 0}, {CS, 2, 0}, /* 28 home */ + {CUL, 2, 0}, {TK, 2, 0}, {TINS, 1, 0}, {C, 0, 1}, {C, 0, 0}, {DIV, 0, 0}, {ST, 0, 0}, /* 30 bad */ + {TL, 1, 0}, {CHR, 0, 0}, /* 37 view chars */ + {CUL, 2, 0}, {CC, 0, 0}, /* 39 view offset */ + {CUL, 2, 0}, {CC, 1, 0}, /* 41 view line_no */ + {CUL, 2, 0}, {TLN, 1, 0}, /* 43 view line */ + {TL, 3, 0}, {TK, 3, 0}, {CAT, 0, 0}, {TL, 1, 0}, {CHR, 0, 0}, {NT, 0, 0}, {CAT, 0, 0}, + {TK, 4, 0}, {CAT, 0, 0}, /* 45 view header */ + {C, 0, 1}, {C, 0, 3}, {DIV, 0, 0}, {NT, 0, 0}, /* 54 view ratio */ + }; + static const H handlers[] = {{1, 0, 7}, {3, 7, 4}, {5, 11, 4}, {7, 15, 4}, {9, 19, 4}, {11, 23, 5}, + {13, 28, 2}, {15, 30, 7}}; + static const V number_views[] = {{"chars", 37, 2}, {"offset", 39, 2}, {"line_no", 41, 2}}; + static const V text_views[] = {{"line", 43, 2}, {"header", 45, 9}, {"ratio", 54, 4}}; + Spec spec = {nums, 1, inputs, LEN(inputs), number_views, 3, handlers, LEN(handlers), code, LEN(code), + NULL, 0, textual, 3, constants, LEN(constants), text_views, 3, NULL, 0, NULL, 0}; + text_spec = spec; + if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + CHECK(!strcmp(state_text(&app, 1), "\xea\xb0\x80" "a\nbc") && app.values[2] == 0); + for (int i = 0; i < 6; ++i) CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK); + CHECK(app.values[2] == 7); /* stops at the end */ + CHECK(number_view(&app, 0) == 5 && number_view(&app, 2) == 2 && !strcmp(view_text(&app, 3), "bc")); + CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 4); /* up keeps column 2 */ + const char *han = "\xed\x95\x9c"; + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)han, 3}) == PURE_OK); + CHECK(!strcmp(state_text(&app, 1), "\xea\xb0\x80" "a\xed\x95\x9c\nbc") && app.values[2] == 7 && app.values[0] == 1); + CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 4); /* back over 3 bytes */ + CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 3); + CHECK(!strcmp(state_text(&app, 1), "\xea\xb0\x80\nbc")); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); /* stops at the start */ + CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); /* nothing before */ + CHECK(pure_app_input(&app, 4, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 4); /* down to column 0 */ + /* A failing event restores the text, the cursor and the states. */ + CHECK(pure_app_input(&app, 7, PURE_EDGE_DOWN) == PURE_INVALID && app.fault == PURE_FAULT_DIVIDE); + CHECK(!strcmp(state_text(&app, 1), "\xea\xb0\x80\nbc") && app.values[2] == 4 && app.values[0] == 1); + CHECK(!strcmp(view_text(&app, 4), "\xeb\xa9\x94\xeb\xaa\xa8 \xc2\xb7 4\xec\x9e\x90")); + CHECK(!strcmp(view_text(&app, 5), "0.3333333333333333")); + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xff", 1}) == PURE_INVALID); + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xed\xa0\x80", 3}) == PURE_INVALID); /* surrogate */ + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xf0\x9f\x98\x80", 4}) == PURE_OK); + CHECK(app.values[2] == 8 && number_view(&app, 1) == 3); /* offset in characters */ + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 4); /* back over the emoji */ + CHECK(pure_app_input(&app, 6, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); + /* Replacement keeps the text and its cursor by name. */ + static const S fresh[] = {{"fresh", 7}}; + static const T kept[] = {{"note", 1, 0}, {"at", 2, 1}}; + static const char *const other[] = {"ignored"}; + Spec next = {fresh, 1, inputs, 0, number_views, 0, handlers, 0, code, 0, NULL, 0, kept, 2, other, 1, text_views, 0, NULL, 0, NULL, 0}; + CHECK(load(next, &module_b) == PURE_OK); + CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 3); + CHECK(pure_app_replace(&app, &module_b) == PURE_OK); + CHECK(app.values[0] == 7 && app.values[2] == 3 && !strcmp(state_text(&app, 1), "\xea\xb0\x80\n\xf0\x9f\x98\x80" "bc")); + pure_app_stop(&app); + + /* Verification of text operations. */ + static const T two[] = {{"one", 1, 0}, {"two", 1, 0}, {"c", 2, 1}}; + static const O wrong_text[] = {{CUL, 3, 0}, {TK, 0, 0}, {TINS, 2, 0}}; + static const H frame[] = {{0, 0, 3}}; + Spec bad = {nums, 1, inputs, 0, number_views, 0, frame, 1, wrong_text, 3, NULL, 0, two, 3, constants, 1, text_views, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "cursor of another text")); + static const O input_in_frame[] = {{CUL, 3, 0}, {TI, 0, 0}, {TINS, 1, 0}}; + bad.o = input_in_frame; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "text input outside")); + static const O store_text[] = {{TK, 0, 0}, {ST, 0, 0}}; + static const H frame2[] = {{0, 0, 2}}; + bad.o = store_text; bad.no = 2; bad.h = frame2; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); + static const O load_text[] = {{L, 1, 0}, {ST, 0, 0}}; + bad.o = load_text; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "no such state")); + static const V text_as_number[] = {{"v", 0, 1}}; + static const O constant_only[] = {{TK, 0, 0}}; + bad = (Spec){nums, 1, inputs, 0, text_as_number, 1, NULL, 0, constant_only, 1, NULL, 0, two, 3, constants, 1, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view must leave one value of its type")); + static const T cursor_on_number[] = {{"c", 2, 0}}; + bad = (Spec){nums, 1, inputs, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, cursor_on_number, 1, constants, 1, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); + static const char *const broken[] = {"\xc0\xaf"}; + bad = (Spec){nums, 1, inputs, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, broken, 1, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "text constant")); +} + +static bool ignore_span(void *context, PureSpan span) { + (void)context; + CHECK(pure_utf8_valid(span) || span.size); + return true; +} + +/* Damaged text modules either fail to load or edit text safely: every cursor + * stays within its text and every text stays valid UTF-8. */ +static void text_mutations(void) { + static unsigned char valid[65536], bytes[65536]; + size_t size = build(valid, text_spec); + static const char *const words[] = {"a", "\xea\xb0\x80", "\n", "\xf0\x9f\x98\x80", "xy\nz", ""}; + uint32_t state = 0x1b873593u; + unsigned accepted = 0; + for (unsigned round = 0; round < 20000; ++round) { + memcpy(bytes, valid, size); + unsigned flips = 1 + next_random(&state) % 3; + for (unsigned f = 0; f < flips; ++f) + bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); + if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; + ++accepted; + PureApp app; + if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; + for (unsigned e = 0; e < 16; ++e) { + uint32_t input = next_random(&state) % 8; + if (module_b.input[input].kind == PURE_KIND_TEXT) { + const char *word = words[next_random(&state) % LEN(words)]; + pure_app_text(&app, input, (PureSpan){(const unsigned char *)word, strlen(word)}); + } else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); + else pure_app_frame(&app); + for (uint32_t v = 0; v < module_b.views; ++v) { + double value; + if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); + else pure_app_view(&app, v, &value); + } + for (uint32_t s = 0; s < module_b.states; ++s) { + if (module_b.state_type[s] == PURE_STATE_CURSOR) { + PureDraft *text = app.texts[module_b.state_ref[s]]; + size_t length = pure_draft_size(text), at = (size_t)app.values[s]; + CHECK(app.values[s] >= 0 && at <= length && app.values[s] == (double)at); + if (at < length) { /* never inside a multi-byte character */ + Text t = {{0}, 0}; + pure_draft_read(text, at, 1, text_span, &t); + CHECK(((unsigned char)t.text[0] & 0xC0) != 0x80); + } + } else if (module_b.state_type[s] == PURE_STATE_TEXT) { + unsigned utf8 = 0; + bool ok = true; + Text t = {{0}, 0}; + if (pure_draft_size(app.texts[s]) < 200) { + pure_draft_read(app.texts[s], 0, pure_draft_size(app.texts[s]), text_span, &t); + ok = pure_utf8_step(&utf8, (PureSpan){(const unsigned char *)t.text, t.size}) && + pure_utf8_complete(utf8); + } + CHECK(ok); + } else CHECK(isfinite(app.values[s])); + } + } + pure_app_stop(&app); + } + printf("{\"text_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); +} + +static Spec list_spec; + +enum { LC = PURE_OP_LIST_COUNT, LA = PURE_OP_LIST_APPEND, LI = PURE_OP_LIST_INSERT, LR = PURE_OP_LIST_REMOVE, + IG = PURE_OP_ITEM_GET, IS = PURE_OP_ITEM_SET, XL = PURE_OP_INDEX_LOAD, XS = PURE_OP_INDEX_STORE, + XM = PURE_OP_INDEX_MOVE, XF = PURE_OP_INDEX_FIRST, XE = PURE_OP_INDEX_LAST, XN = PURE_OP_INDEX_NUM, + NOT = PURE_OP_NOT, TITLE = 1, DONE = 1 | 1u << 16 }; + +static const char *item_title(const PureApp *app, uint32_t list, size_t at) { + static char title[64]; + PureCell cell = pure_list_get(app->lists[list], at, 0); + CHECK(cell.size < sizeof(title)); + if (cell.size) memcpy(title, cell.text, cell.size); + title[cell.size] = 0; + return title; +} + +/* A task list: made (num), tasks (list of task {title, done}), at (index of + * tasks). Indices follow their item across inserts and removes; a failed + * event undoes every list change. */ +static void lists(void) { + static const S nums[] = {{"made", 0}}; + static const T states[] = {{"tasks", 3, 0}, {"at", 4, 1}}; + static const RF task_fields[] = {{"title", 1, 0}, {"done", 0, 0}}; + static const R records[] = {{"task", task_fields, 2}}; + static const char *const constants[] = {"", "new", "x"}; + static const I inputs[] = { + {"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("ins", "KeyI"), KEY("del", "Delete"), + KEY("up", "ArrowUp"), KEY("down", "ArrowDown"), KEY("toggle", "Space"), KEY("rename", "KeyR"), KEY("bad", "KeyB"), + }; + static const O code[] = { + {TI, 0, 0}, {C, 0, 0}, {LA, 1, 0}, {XE, 1, 0}, {XS, 2, 0}, {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, /* 0 typed */ + {XL, 2, 0}, {TK, 1, 0}, {C, 0, 0}, {LI, 1, 0}, /* 9 ins */ + {XL, 2, 0}, {LR, 1, 0}, /* 13 del */ + {XL, 2, 0}, {C, 0, -1}, {XM, 0, 0}, {XS, 2, 0}, /* 15 up */ + {XL, 2, 0}, {C, 0, 1}, {XM, 0, 0}, {XS, 2, 0}, /* 19 down */ + {XL, 2, 0}, {XL, 2, 0}, {IG, DONE, 0}, {NOT, 0, 0}, {IS, DONE, 0}, /* 23 toggle */ + {XL, 2, 0}, {XL, 2, 0}, {IG, TITLE, 0}, {TK, 2, 0}, {CAT, 0, 0}, {IS, TITLE, 0}, /* 28 rename */ + {XL, 2, 0}, {TK, 2, 0}, {IS, TITLE, 0}, {XL, 2, 0}, {LR, 1, 0}, {TK, 1, 0}, {C, 0, 0}, /* 34 bad */ + {LA, 1, 0}, {C, 0, 1}, {C, 0, 0}, {DIV, 0, 0}, {ST, 0, 0}, + {LC, 1, 0}, /* 46 view count */ + {XL, 2, 0}, {XN, 0, 0}, /* 47 view position */ + {XL, 2, 0}, {IG, DONE, 0}, /* 49 view done */ + {XL, 2, 0}, {IG, TITLE, 0}, /* 51 view current */ + }; + static const H handlers[] = {{1, 0, 9}, {3, 9, 4}, {5, 13, 2}, {7, 15, 4}, {9, 19, 4}, {11, 23, 5}, + {13, 28, 6}, {15, 34, 12}}; + static const V number_views[] = {{"count", 46, 1}, {"position", 47, 2}, {"done", 49, 2}}; + static const V text_views[] = {{"current", 51, 2}}; + Spec spec = {nums, 1, inputs, LEN(inputs), number_views, 3, handlers, LEN(handlers), code, LEN(code), + NULL, 0, states, 2, constants, 3, text_views, 1, records, 1, NULL, 0}; + list_spec = spec; + if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + double value; + CHECK(number_view(&app, 0) == 0 && number_view(&app, 1) == 0 && !pure_app_view(&app, 2, &value)); + CHECK(pure_app_view_text(&app, 3, text_span, &(Text){{0}, 0}) == PURE_NOT_FOUND); /* no item yet */ + CHECK(app.fault == PURE_FAULT_NONE); /* views leave faults alone */ + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"a", 1}) == PURE_OK); + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"b", 1}) == PURE_OK); + CHECK(pure_list_count(app.lists[1]) == 2 && app.values[2] == 1 && app.values[0] == 2); + CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); + CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); /* stops at the first */ + /* Inserting before the current item: the index follows it. */ + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 1); + CHECK(!strcmp(item_title(&app, 1, 0), "new") && !strcmp(view_text(&app, 3), "a")); + CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && number_view(&app, 2) == 1); + CHECK(pure_app_input(&app, 6, PURE_EDGE_DOWN) == PURE_OK && !strcmp(view_text(&app, 3), "ax")); + CHECK(pure_app_input(&app, 4, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 2); + CHECK(pure_app_input(&app, 4, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 2); /* stops at the last */ + /* Removing the last item: the index moves to the new last. */ + CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 1 && pure_list_count(app.lists[1]) == 2); + CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); + /* Removing the current item: the index names the next. */ + CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0 && !strcmp(view_text(&app, 3), "ax")); + /* A failing event undoes a set, a remove and an append. */ + CHECK(pure_app_input(&app, 7, PURE_EDGE_DOWN) == PURE_INVALID && app.fault == PURE_FAULT_DIVIDE); + CHECK(pure_list_count(app.lists[1]) == 1 && !strcmp(item_title(&app, 1, 0), "ax") && + pure_list_get(app.lists[1], 0, 1).num == 1 && app.values[2] == 0 && app.values[0] == 2); + CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && pure_list_count(app.lists[1]) == 0 && app.values[2] == 0); + CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_NOT_FOUND && app.fault == PURE_FAULT_NOT_FOUND); + CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_NOT_FOUND); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && pure_list_count(app.lists[1]) == 1 && app.values[2] == 0); + for (int i = 0; i < 3; ++i) CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xea\xb0\x80", 3}) == PURE_OK); + CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 2); + CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK); + + /* Replacement keeps the list by name. The same record keeps the list + * itself; a changed record keeps each field by name and kind and gives new + * fields their default. */ + static const S no_nums[] = {{"made", 0}}; + Spec same = {no_nums, 1, inputs, 0, number_views, 0, handlers, 0, code, 0, NULL, 0, states, 2, constants, 3, + text_views, 0, records, 1, NULL, 0}; + CHECK(load(same, &module_b) == PURE_OK); + const PureList *before = app.lists[1]; + CHECK(pure_app_replace(&app, &module_b) == PURE_OK && app.lists[1] == before && app.values[2] == 2); + static const RF changed_fields[] = {{"done", 0, 0}, {"note", 1, 1}, {"title", 0, 7}, {"rank", 0, 5}}; + static const R changed[] = {{"task", changed_fields, 4}}; + Spec next = same; + next.r = changed; + CHECK(load(next, &module_a) == PURE_OK); + CHECK(pure_app_replace(&app, &module_a) == PURE_OK && pure_list_count(app.lists[1]) == 4 && app.values[2] == 2); + PureCell cell = pure_list_get(app.lists[1], 2, 0); + CHECK(cell.num == 1); /* done kept by name */ + cell = pure_list_get(app.lists[1], 2, 1); + CHECK(cell.size == 3 && !memcmp(cell.text, "new", 3)); /* note: its default */ + CHECK(pure_list_get(app.lists[1], 2, 2).num == 7); /* title is now a number: default */ + CHECK(pure_list_get(app.lists[1], 0, 3).num == 5); + /* A state whose kind changed starts again. */ + static const T swapped_kinds[] = {{"tasks", 1, 0}, {"at", 3, 0}}; + next.t = swapped_kinds; + CHECK(load(next, &module_b) == PURE_OK); + CHECK(pure_app_replace(&app, &module_b) == PURE_OK && !app.lists[1] && pure_draft_size(app.texts[1]) == 0 && + pure_list_count(app.lists[2]) == 0); + pure_app_stop(&app); + + /* Verification of list operations. */ + static const T two_lists[] = {{"a", 3, 0}, {"b", 3, 0}, {"ia", 4, 1}}; + static const O other_index[] = {{XL, 3, 0}, {LR, 2, 0}}; + static const H frame[] = {{0, 0, 2}}; + Spec bad = {nums, 1, inputs, 0, number_views, 0, frame, 1, other_index, 2, NULL, 0, two_lists, 3, constants, 3, + text_views, 0, records, 1, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "index of another list")); + static const O swapped[] = {{C, 0, 0}, {TK, 0, 0}, {LA, 1, 0}}; + static const H frame3[] = {{0, 0, 3}}; + bad.o = swapped; bad.no = 3; bad.h = frame3; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); + static const O short_record[] = {{TK, 0, 0}, {LA, 1, 0}}; + bad.o = short_record; bad.no = 2; bad.h = frame; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "stack underflow")); + static const O no_field[] = {{XL, 3, 0}, {IG, 1 | 2u << 16, 0}, {ST, 0, 0}}; + bad.o = no_field; bad.no = 3; bad.h = frame3; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "no such state")); + static const O index_as_number[] = {{XL, 3, 0}, {ST, 0, 0}}; + bad.o = index_as_number; bad.no = 2; bad.h = frame; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); + static const V append_view[] = {{"v", 0, 3}}; + bad = (Spec){nums, 1, inputs, 0, append_view, 1, NULL, 0, (O[]){{TK, 0, 0}, {C, 0, 0}, {LA, 1, 0}}, 3, NULL, 0, + two_lists, 3, constants, 3, NULL, 0, records, 1, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view with an effect")); + static const T index_of_number[] = {{"ia", 4, 0}}; + bad = (Spec){nums, 1, inputs, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, index_of_number, 1, constants, 3, NULL, 0, + records, 1, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); + static const T list_of_nothing[] = {{"a", 3, 1}}; + bad.t = list_of_nothing; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); + bad.t = NULL; bad.nt = 0; + static const RF repeated[] = {{"x", 0, 0}, {"x", 1, 0}}; + bad.r = (R[]){{"task", repeated, 2}}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "duplicate record field")); + static const RF far_default[] = {{"x", 1, 3}}; + bad.r = (R[]){{"task", far_default, 1}}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "record field default")); + static const RF infinite[] = {{"x", 0, INFINITY}}; + bad.r = (R[]){{"task", infinite, 1}}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "record field default")); + bad.r = (R[]){{"task", NULL, 0}}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "record")); + bad.r = (R[]){{"made", task_fields, 2}}; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "duplicate name")); +} + +/* Damaged list modules either fail to load or keep every index on an item + * of its list and every text field valid UTF-8. */ +static void list_mutations(void) { + static unsigned char valid[65536], bytes[65536]; + size_t size = build(valid, list_spec); + static const char *const words[] = {"a", "\xea\xb0\x80", "\n", ""}; + uint32_t state = 0x85ebca6bu; + unsigned accepted = 0; + for (unsigned round = 0; round < 20000; ++round) { + memcpy(bytes, valid, size); + unsigned flips = 1 + next_random(&state) % 3; + for (unsigned f = 0; f < flips; ++f) + bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); + if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; + ++accepted; + PureApp app; + if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; + for (unsigned e = 0; e < 24; ++e) { + uint32_t input = next_random(&state) % 8; + if (module_b.input[input].kind == PURE_KIND_TEXT) { + const char *word = words[next_random(&state) % LEN(words)]; + pure_app_text(&app, input, (PureSpan){(const unsigned char *)word, strlen(word)}); + } else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); + else pure_app_frame(&app); + for (uint32_t v = 0; v < module_b.views; ++v) { + double value; + if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); + else pure_app_view(&app, v, &value); + } + for (uint32_t s = 0; s < module_b.states; ++s) { + if (module_b.state_type[s] == PURE_STATE_INDEX) { + size_t count = pure_list_count(app.lists[module_b.state_ref[s]]); + CHECK(app.values[s] == floor(app.values[s]) && app.values[s] >= 0 && + app.values[s] <= (count ? (double)(count - 1) : 0)); + } else if (module_b.state_type[s] == PURE_STATE_LIST) { + const PureRecord *record = &module_b.record[module_b.state_ref[s]]; + for (size_t at = 0; at < pure_list_count(app.lists[s]); ++at) + for (unsigned f = 0; f < record->count; ++f) { + PureCell cell = pure_list_get(app.lists[s], at, f); + if (module_b.record_field_text[record->first + f]) + CHECK(pure_utf8_valid((PureSpan){cell.text, cell.size})); + else CHECK(isfinite(cell.num)); + } + } else if (module_b.state_type[s] == PURE_STATE_NUM) CHECK(isfinite(app.values[s])); + } + } + pure_app_stop(&app); + } + printf("{\"list_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); +} + +static Spec table_spec; + +static const char *cell_text(PureApp *app, uint32_t column, size_t row) { + static Text t; + t.size = 0; + t.text[0] = 0; + CHECK(pure_app_cell_text(app, column, row, text_span, &t) == PURE_OK); + return t.text; +} + +enum { ROW = PURE_OP_ROW, EQ = PURE_OP_EQ, LT = PURE_OP_LT, PICK = PURE_OP_PICK, SLIDE = PURE_OP_SLIDE }; + +/* A table view has a row per item; its columns read the row and may branch. + * A value crosses a join when every path brings the same stack. */ +static void tables(void) { + static const S nums[] = {{"n", 0}}; + static const T states[] = {{"tasks", 3, 0}, {"at", 4, 1}}; + static const RF task_fields[] = {{"title", 1, 0}, {"done", 0, 0}}; + static const R records[] = {{"task", task_fields, 2}}; + static const char *const constants[] = {"", "x", " ", "empty"}; + static const I inputs[] = {{"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("toggle", "Space")}; + static const O code[] = { + {TI, 0, 0}, {C, 0, 0}, {LA, 1, 0}, {XE, 1, 0}, {XS, 2, 0}, /* 0 typed */ + {XL, 2, 0}, {XL, 2, 0}, {IG, DONE, 0}, {NOT, 0, 0}, {IS, DONE, 0}, /* 5 toggle */ + {LC, 1, 0}, {C, 0, 0}, {GT, 0, 0}, {JZ, 17, 0}, {XL, 2, 0}, {IG, TITLE, 0}, {JMP, 18, 0}, + {TK, 3, 0}, /* 10 current */ + {ROW, 1, 0}, {IG, TITLE, 0}, /* 18 title */ + {ROW, 1, 0}, {IG, DONE, 0}, {JZ, 25, 0}, {TK, 1, 0}, {JMP, 26, 0}, {TK, 2, 0}, /* 20 mark */ + {ROW, 1, 0}, {XN, 0, 0}, /* 26 pos */ + {ROW, 1, 0}, {XN, 0, 0}, {XL, 2, 0}, {XN, 0, 0}, {EQ, 0, 0}, /* 28 here */ + }; + static const H handlers[] = {{1, 0, 5}, {3, 5, 5}}; + static const V text_views[] = {{"current", 10, 8}}; + static const W table[] = {{"board", 2, 0, 1, 0}, {"title", 1, 2, 18, 2}, {"mark", 1, 2, 20, 6}, + {"pos", 0, 2, 26, 2}, {"here", 0, 2, 28, 5}}; + Spec spec = {nums, 1, inputs, 2, NULL, 0, handlers, 2, code, LEN(code), NULL, 0, states, 2, constants, 4, + text_views, 1, records, 1, table, 5}; + table_spec = spec; + if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } + CHECK(pure_module_view(&module_a, "board") == 1 && pure_module_view(&module_a, "title") == -1 && + pure_module_column(&module_a, 1, "title") == 2 && pure_module_column(&module_a, 1, "current") == -1); + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + double value; + CHECK(pure_app_rows(&app, 1) == 0 && !strcmp(view_text(&app, 0), "empty")); /* the else branch */ + CHECK(pure_app_cell_text(&app, 2, 0, text_span, &(Text){{0}, 0}) == PURE_NOT_FOUND); + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"a", 1}) == PURE_OK); + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xeb\xb9\xb5", 3}) == PURE_OK); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); + CHECK(pure_app_rows(&app, 1) == 2 && !strcmp(view_text(&app, 0), "\xeb\xb9\xb5")); + CHECK(!strcmp(cell_text(&app, 2, 0), "a") && !strcmp(cell_text(&app, 2, 1), "\xeb\xb9\xb5")); + CHECK(!strcmp(cell_text(&app, 3, 0), " ") && !strcmp(cell_text(&app, 3, 1), "x")); + CHECK(pure_app_cell(&app, 4, 1, &value) && value == 1 && pure_app_cell(&app, 5, 0, &value) && value == 0 && + pure_app_cell(&app, 5, 1, &value) && value == 1); + CHECK(!pure_app_cell(&app, 4, 2, &value) && pure_app_cell_text(&app, 2, 2, text_span, &(Text){{0}, 0}) == PURE_NOT_FOUND); + /* Columns are read by row, other views without one. */ + CHECK(!pure_app_view(&app, 4, &value) && pure_app_view_text(&app, 2, text_span, &(Text){{0}, 0}) == PURE_INVALID); + CHECK(pure_app_cell_text(&app, 0, 0, text_span, &(Text){{0}, 0}) == PURE_INVALID && pure_app_rows(&app, 0) == 0); + CHECK(pure_app_view_text(&app, 1, text_span, &(Text){{0}, 0}) == PURE_INVALID); + pure_app_stop(&app); + + /* Verification of rows, tables and joins. */ + Spec bad = spec; + static const V row_view[] = {{"current", 18, 2}}; + bad.tv = row_view; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "row outside a column")); + static const H row_handler[] = {{3, 26, 2}}; + bad = spec; bad.h = row_handler; bad.nh = 1; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "row outside a column")); + static const W twice[] = {{"board", 2, 0, 1, 0}, {"title", 1, 2, 18, 2}, {"title", 1, 2, 18, 2}}; + bad = spec; bad.w = twice; bad.nw = 3; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "duplicate column")); + static const W two_tables[] = {{"board", 2, 0, 1, 0}, {"title", 1, 2, 18, 2}, {"other", 2, 0, 1, 0}, + {"title", 1, 4, 18, 2}, {"tasks", 0, 4, 26, 2}}; + bad = spec; bad.w = two_tables; bad.nw = 5; + CHECK(load(bad, &module_b) == PURE_OK); /* column names belong to their table */ + static const W not_a_table[] = {{"title", 1, 1, 18, 2}}; + bad = spec; bad.w = not_a_table; bad.nw = 1; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view type")); + static const W over_number[] = {{"board", 2, 0, 0, 0}}; + bad = spec; bad.w = over_number; bad.nw = 1; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "table list")); + static const O other_row[] = {{ROW, 2, 0}, {XN, 0, 0}}; + static const W other_row_column[] = {{"board", 2, 0, 1, 0}, {"pos", 0, 1, 0, 2}}; + bad = spec; bad.o = other_row; bad.no = 2; bad.h = NULL; bad.nh = 0; bad.tv = NULL; bad.ntv = 0; + bad.w = other_row_column; bad.nw = 2; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "no such state")); + static const O typed_join[] = {{L, 0, 0}, {JZ, 4, 0}, {TK, 0, 0}, {JMP, 5, 0}, {C, 0, 1}}; + static const V mixed[] = {{"current", 0, 5}}; + bad = spec; bad.o = typed_join; bad.no = 5; bad.h = NULL; bad.nh = 0; bad.tv = mixed; bad.w = NULL; bad.nw = 0; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "join with different stacks")); + /* At most 8 joins with a value may be ahead at once. */ + for (unsigned open = 8; open <= 9; ++open) { + static O nested[1 + 2 * 9 + 9 + 1]; + unsigned n = 0; + nested[n++] = (O){C, 0, 1}; + for (unsigned k = 0; k < open; ++k) { nested[n++] = (O){C, 0, 0}; nested[n++] = (O){JZ, 1 + 2 * open + k, 0}; } + for (unsigned k = 0; k < open; ++k) nested[n++] = (O){NEG, 0, 0}; + nested[n++] = (O){ST, 0, 0}; + H frame = {0, 0, n}; + bad = spec; bad.o = nested; bad.no = n; bad.h = &frame; bad.nh = 1; bad.tv = NULL; bad.ntv = 0; bad.w = NULL; bad.nw = 0; + CHECK(load(bad, &module_b) == (open == 8 ? PURE_OK : PURE_INVALID)); + } + CHECK(strstr(error, "more than 8 joins ahead")); + + /* PICK copies a value from below, SLIDE drops values under the top: a + * value used twice is computed once. (3 + 4) * (3 + 4) with a text between. */ + static const O picked[] = {{C, 0, 3}, {C, 0, 4}, {ADD, 0, 0}, {TK, 1, 0}, {PICK, 1, 0}, {PICK, 2, 0}, {MUL, 0, 0}, + {SLIDE, 2, 0}}; + static const V squared[] = {{"sq", 0, 8}}; + bad = (Spec){nums, 1, inputs, 0, squared, 1, NULL, 0, picked, 8, NULL, 0, NULL, 0, constants, 2, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(bad, &module_b) == PURE_OK); + CHECK(pure_app_start(&app, &module_b, (PureData){0}) == PURE_OK && number_view(&app, 0) == 49); + pure_app_stop(&app); + static const O pick_text[] = {{TK, 1, 0}, {C, 0, 1}, {PICK, 1, 0}, {SLIDE, 2, 0}}; + static const V text_left[] = {{"t", 0, 4}}; + bad.v = NULL; bad.nv = 0; bad.tv = text_left; bad.ntv = 1; bad.o = pick_text; bad.no = 4; + CHECK(load(bad, &module_b) == PURE_OK); + CHECK(pure_app_start(&app, &module_b, (PureData){0}) == PURE_OK && !strcmp(view_text(&app, 0), "x")); + pure_app_stop(&app); + static const O pick_type[] = {{TK, 1, 0}, {PICK, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {SLIDE, 1, 0}}; + static const V five[] = {{"sq", 0, 5}}, three[] = {{"sq", 0, 3}}; + bad.o = pick_type; bad.no = 5; bad.tv = NULL; bad.ntv = 0; bad.v = five; bad.nv = 1; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); /* a copy keeps its type */ + static const O pick_far[] = {{C, 0, 1}, {PICK, 1, 0}, {SLIDE, 1, 0}}; + bad.o = pick_far; bad.no = 3; bad.v = three; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "stack underflow")); + static const O slide_all[] = {{C, 0, 1}, {C, 0, 2}, {SLIDE, 2, 0}}; + bad.o = slide_all; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "stack underflow")); + static const O slide_none[] = {{C, 0, 1}, {C, 0, 2}, {SLIDE, 0, 0}}; + bad.o = slide_none; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "stack underflow")); +} + +/* Damaged table modules either fail to load or evaluate every cell safely. */ +static void table_mutations(void) { + static unsigned char valid[65536], bytes[65536]; + size_t size = build(valid, table_spec); + uint32_t state = 0xc2b2ae35u; + unsigned accepted = 0; + for (unsigned round = 0; round < 20000; ++round) { + memcpy(bytes, valid, size); + unsigned flips = 1 + next_random(&state) % 3; + for (unsigned f = 0; f < flips; ++f) + bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); + if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; + ++accepted; + PureApp app; + if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; + for (unsigned e = 0; e < 12; ++e) { + uint32_t input = next_random(&state) % 2; + if (module_b.input[input].kind == PURE_KIND_TEXT) pure_app_text(&app, input, (PureSpan){(const unsigned char *)"t", 1}); + else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); + for (uint32_t v = 0; v < module_b.views; ++v) { + double value; + if (!module_b.view_table[v]) { + if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); + else pure_app_view(&app, v, &value); + continue; + } + size_t rows = pure_app_rows(&app, module_b.view_table[v] - 1u); + for (size_t row = 0; row <= rows; ++row) { + if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_cell_text(&app, v, row, ignore_span, NULL); + else if (pure_app_cell(&app, v, row, &value)) CHECK(isfinite(value)); + } + } + } + pure_app_stop(&app); + } + printf("{\"table_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); +} + +/* ---- Stored states ---------------------------------------------------- */ + +static char *new_store(void) { + static char path[64]; + snprintf(path, sizeof(path), "/tmp/pure-module-store-XXXXXX"); + CHECK(mkdtemp(path) && pure_store_init(path) == PURE_OK); + return path; +} + +static void remove_store(const char *path) { + DIR *entries = opendir(path); + CHECK(entries); + struct dirent *entry; + while ((entry = readdir(entries))) + if (strcmp(entry->d_name, ".") && strcmp(entry->d_name, "..")) CHECK(unlinkat(dirfd(entries), entry->d_name, 0) == 0); + CHECK(closedir(entries) == 0 && rmdir(path) == 0); +} + +typedef struct { unsigned char bytes[4096]; size_t size; } Bytes; + +static bool bytes_span(void *context, PureSpan span) { + Bytes *b = context; + CHECK(b->size + span.size <= sizeof(b->bytes)); + memcpy(b->bytes + b->size, span.data, span.size); + b->size += span.size; + return true; +} + +/* Two apps of one module hold equal states: numbers bit for bit, texts and + * list items byte for byte. */ +static void same_states(const PureApp *a, const PureApp *b) { + const PureModule *m = a->module; + for (uint32_t i = 0; i < m->states; ++i) { + if (m->state_type[i] == PURE_STATE_TEXT) { + static Bytes x, y; + x.size = y.size = 0; + CHECK(pure_draft_read(a->texts[i], 0, pure_draft_size(a->texts[i]), bytes_span, &x) == PURE_OK); + CHECK(pure_draft_read(b->texts[i], 0, pure_draft_size(b->texts[i]), bytes_span, &y) == PURE_OK); + CHECK(x.size == y.size && !memcmp(x.bytes, y.bytes, x.size)); + } else if (m->state_type[i] == PURE_STATE_LIST) { + const PureRecord *r = &m->record[m->state_ref[i]]; + CHECK(pure_list_count(a->lists[i]) == pure_list_count(b->lists[i])); + for (size_t at = 0; at < pure_list_count(a->lists[i]); ++at) + for (unsigned f = 0; f < r->count; ++f) { + PureCell x = pure_list_get(a->lists[i], at, f), y = pure_list_get(b->lists[i], at, f); + CHECK(!memcmp(&x.num, &y.num, sizeof(x.num)) && x.size == y.size && (!x.size || !memcmp(x.text, y.text, x.size))); + } + } else CHECK(!memcmp(&a->values[i], &b->values[i], sizeof(double))); + } +} + +/* The same random events on an app in memory and on one whose states are + * stored and which restarts from them every few events: equal after every + * step, failed events included. */ +static void stored_equivalence(Spec spec, const char *const *words, size_t word_count, uint32_t seed) { + char *path = new_store(); + CHECK(load(spec, &module_a) == PURE_OK); + PureApp memory, stored; + CHECK(pure_app_start(&memory, &module_a, (PureData){0}) == PURE_OK); + CHECK(pure_app_start(&stored, &module_a, pure_tree_data(path)) == PURE_OK); + unsigned restarts = 0; + for (unsigned step = 0; step < 400; ++step) { + uint32_t input = next_random(&seed) % module_a.inputs; + PureResult a, b; + if (module_a.input[input].kind == PURE_KIND_TEXT) { + const char *word = words[next_random(&seed) % word_count]; + PureSpan text = {(const unsigned char *)word, strlen(word)}; + a = pure_app_text(&memory, input, text); + b = pure_app_text(&stored, input, text); + } else { + a = pure_app_input(&memory, input, PURE_EDGE_DOWN); + b = pure_app_input(&stored, input, PURE_EDGE_DOWN); + } + CHECK(a == b); + same_states(&memory, &stored); + if (next_random(&seed) % 7 == 0) { + pure_app_stop(&stored); + CHECK(pure_app_start(&stored, &module_a, pure_tree_data(path)) == PURE_OK); + same_states(&memory, &stored); + ++restarts; + } + } + CHECK(restarts > 20); + pure_app_stop(&memory); + pure_app_stop(&stored); + remove_store(path); +} + +/* One event changes a number, a text and a list; the process dies at each + * commit stage. Starting again finds all three as before or all as after. */ +enum { NOTE = 1, TASKS = 3 }; +static const S crash_nums[] = {{"n", 0}}; +static const T crash_states[] = {{"note", 1, 0}, {"at", 2, 1}, {"tasks", 3, 0}, {"i", 4, 3}}; +static const RF crash_fields[] = {{"title", 1, 0}, {"done", 0, 0}}; +static const R crash_records[] = {{"task", crash_fields, 2}}; +static const char *const crash_constants[] = {"", "x"}; +static const I crash_inputs[] = {KEY("go", "KeyG")}; +static const O crash_code[] = { + {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, /* n = n + 1 */ + {CUL, 2, 0}, {TK, 1, 0}, {TINS, 1, 0}, /* note.insert(at, "x") */ + {TK, 1, 0}, {L, 0, 0}, {LA, 3, 0}, /* tasks.append(task { "x", n }) */ + {XE, 3, 0}, {XS, 4, 0}, /* i = tasks.last */ +}; +static const H crash_handlers[] = {{1, 0, 12}}; + +static unsigned consistent(const PureApp *app) { + unsigned n = (unsigned)app->values[0]; + CHECK(app->values[0] == n && pure_draft_size(app->texts[NOTE]) == n && app->values[2] == n); + CHECK(pure_list_count(app->lists[TASKS]) == n && app->values[4] == (n ? n - 1 : 0)); + for (unsigned k = 0; k < n; ++k) CHECK(pure_list_get(app->lists[TASKS], k, 1).num == k + 1); + return n; +} + +static void stored_crashes(void) { + char *path = new_store(); + Spec spec = {crash_nums, 1, crash_inputs, 1, NULL, 0, crash_handlers, 1, crash_code, LEN(crash_code), NULL, 0, + crash_states, 4, crash_constants, 2, NULL, 0, crash_records, 1, NULL, 0}; + CHECK(load(spec, &module_a) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); + CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK && consistent(&app) == 1); + pure_app_stop(&app); + unsigned expected = 1; + for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + PureApp doomed; + if (pure_app_start(&doomed, &module_a, pure_tree_data(path)) != PURE_OK) _exit(1); + crash = 1; + fail_stage = stage; + pure_app_input(&doomed, 0, PURE_EDGE_DOWN); + _exit(0); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); + if (stage == PURE_BEFORE_DIR_SYNC) ++expected; /* the head was written */ + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == expected); + pure_app_stop(&app); + } + /* A failed commit leaves the app and the store as they were; after an + * uncertain one the app no longer matches the store and its events fail + * until it starts again, from what the store has. */ + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); + for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; ++fail_stage) { + CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_STORE_ERROR && consistent(&app) == expected); + } + fail_stage = PURE_BEFORE_DIR_SYNC; + CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_UNCERTAIN && consistent(&app) == expected); + fail_stage = -1; + CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_CONFLICT && consistent(&app) == expected); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == expected + 1); + CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK && consistent(&app) == expected + 2); + pure_app_stop(&app); + remove_store(path); +} + +/* Frame changes are stored by the next event or by a save; a stop alone + * leaves them out. */ +static void stored_frames(void) { + char *path = new_store(); + static const O code[] = {{L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}}; + static const H handlers[] = {{0, 0, 4}, {1, 0, 0}}; + Spec spec = {crash_nums, 1, crash_inputs, 1, NULL, 0, handlers, 2, code, 4, NULL, 0, NULL, 0, NULL, 0, NULL, 0, + NULL, 0, NULL, 0}; + CHECK(load(spec, &module_a) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); + for (int i = 0; i < 5; ++i) CHECK(pure_app_frame(&app) == PURE_OK); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 0); + for (int i = 0; i < 3; ++i) CHECK(pure_app_frame(&app) == PURE_OK); + CHECK(pure_app_save(&app) == PURE_OK); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 3); + for (int i = 0; i < 2; ++i) CHECK(pure_app_frame(&app) == PURE_OK); + CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 5); + pure_app_stop(&app); + remove_store(path); +} + +static bool list_keys(void *context, const char *key) { + strcat(context, key); + strcat(context, ","); + return true; +} + +/* A message's document and the states its handlers change commit together. */ +static void stored_messages(void) { + char *path = new_store(); + CHECK(load(editor_spec, &module_a) == PURE_OK); + PureApp app; + unsigned char buffer[64]; + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); + CHECK(pure_app_message(&app, 1, "doc", 0, edit_payload(buffer, 0, 0, "hello")) == PURE_OK && app.revision == 1); + CHECK(app.values[0] == 1); + CHECK(pure_app_message(&app, 1, "doc", 0, edit_payload(buffer, 0, 0, "x")) == PURE_CONFLICT && app.values[0] == 1); + /* An edit past the size guard fails: neither the document nor the count moves. */ + CHECK(pure_app_message(&app, 3, "doc", 1, edit_payload(buffer, 5, 0, "-world")) == PURE_LIMIT && app.values[0] == 1); + CHECK(pure_app_message(&app, 3, "doc", 1, edit_payload(buffer, 5, 0, "!")) == PURE_OK && app.values[0] == 2); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 2); + CHECK(pure_app_message(&app, 0, "doc", -1, (PureSpan){0}) == PURE_OK && app.revision == 2 && replied(&app, "hello!")); + pure_app_stop(&app); + /* Documents stay documents: the app's states are not among the keys. */ + char seen[256] = ""; + CHECK(pure_store_keys(path, list_keys, seen) == PURE_OK && !strcmp(seen, "doc,")); + remove_store(path); +} + +/* A stored list whose record changed keeps each field by name, when the app + * starts from it and when its module is replaced; stored values that no + * longer fit fall back, and damaged stored values stop the start. */ +static void stored_migration(void) { + char *path = new_store(); + static const RF changed_fields[] = {{"done", 0, 0}, {"note", 1, 1}, {"title", 1, 0}, {"rank", 0, 5}}; + static const R changed[] = {{"task", changed_fields, 4}}; + Spec before = {crash_nums, 1, crash_inputs, 1, NULL, 0, crash_handlers, 1, crash_code, LEN(crash_code), NULL, 0, + crash_states, 4, crash_constants, 2, NULL, 0, crash_records, 1, NULL, 0}; + Spec after = before; + after.r = changed; + after.h = NULL; + after.nh = 0; + CHECK(load(before, &module_a) == PURE_OK && load(after, &module_b) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); + CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK && pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); + pure_app_stop(&app); + for (int round = 0; round < 2; ++round) { /* migrated, then stored as migrated */ + CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK && pure_list_count(app.lists[TASKS]) == 2); + PureCell note = pure_list_get(app.lists[TASKS], 1, 1), title = pure_list_get(app.lists[TASKS], 1, 2); + CHECK(pure_list_get(app.lists[TASKS], 1, 0).num == 2 && pure_list_get(app.lists[TASKS], 1, 3).num == 5); + CHECK(note.size == 1 && note.text[0] == 'x' && title.size == 1 && title.text[0] == 'x'); + CHECK(app.values[0] == 2 && app.values[4] == 1 && pure_draft_size(app.texts[NOTE]) == 2); + pure_app_stop(&app); + } + /* Back to the first record by replacing the module; the next save stores it. */ + CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK); + CHECK(pure_app_replace(&app, &module_a) == PURE_OK && pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == 3); + pure_app_stop(&app); + /* Stored numbers that no longer fit their state start again. */ + PureDraft *values; + int64_t revision; + CHECK(pure_state_open(path, "values", &values) == PURE_OK); + unsigned char record[64] = "PVS1"; + size_t at = 4; + const char *names[] = {"at", "i", "n"}; + double stored[] = {100, 7.5, 9}; + uint8_t types[] = {PURE_STATE_CURSOR, PURE_STATE_INDEX, PURE_STATE_NUM}; + for (unsigned k = 0; k < 3; ++k) { + record[at++] = types[k]; + record[at++] = (unsigned char)strlen(names[k]); + memcpy(record + at, names[k], strlen(names[k])); + at += strlen(names[k]); + uint64_t bits; + memcpy(&bits, &stored[k], sizeof(bits)); + for (unsigned b = 0; b < 8; ++b) record[at++] = (unsigned char)(bits >> (8 * b)); + } + CHECK(pure_draft_splice(values, 0, pure_draft_size(values), (PureSpan){record, at}) == PURE_OK); + CHECK(pure_draft_commit(values, &revision) == PURE_OK); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); + CHECK(app.values[0] == 9 && app.values[2] == 0 && app.values[4] == 0); + pure_app_stop(&app); + /* Damaged stored lists and texts are refused, not replaced. */ + const char *damaged[] = {"list-tasks", "text-note"}; + for (unsigned k = 0; k < 2; ++k) { + PureDraft *draft; + CHECK(pure_state_open(path, damaged[k], &draft) == PURE_OK); + CHECK(pure_draft_splice(draft, 0, 0, (PureSpan){(const unsigned char *)"\xff", 1}) == PURE_OK); + CHECK(pure_draft_commit(draft, &revision) == PURE_OK); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_STORE_ERROR); + CHECK(pure_draft_splice(draft, 0, 1, (PureSpan){0}) == PURE_OK && pure_draft_commit(draft, &revision) == PURE_OK); + pure_draft_close(draft); + } + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); + CHECK(pure_list_count(app.lists[TASKS]) == 3 && pure_draft_size(app.texts[NOTE]) == 3 && app.values[0] == 9); + pure_app_stop(&app); + pure_draft_close(values); + remove_store(path); +} + +static void stored_states(void) { + static const char *const list_words[] = {"a", "\xea\xb0\x80", "", "\n", "\xf0\x9f\x98\x80"}; + static const char *const text_words[] = {"a", "\xea\xb0\x80", "\n", "\xf0\x9f\x98\x80", "xy\nz", ""}; + stored_equivalence(list_spec, list_words, LEN(list_words), 0x9e3779b9u); + stored_equivalence(text_spec, text_words, LEN(text_words), 0x7f4a7c15u); + stored_crashes(); + stored_frames(); + stored_messages(); + stored_migration(); +} + +/* ---- Deferred saving ---------------------------------------------------- */ + +typedef struct { uint32_t input; const char *word; } Step; + +static PureResult apply_step(PureApp *app, Step step) { + if (step.word) return pure_app_text(app, step.input, (PureSpan){(const unsigned char *)step.word, strlen(step.word)}); + return pure_app_input(app, step.input, PURE_EDGE_DOWN); +} + +/* Deferred events apply at once and last only through a save: an app that + * stops without saving comes back as it was at the last save. The oracle is + * an app in memory that replays exactly the saved events. */ +static void deferred_equivalence(Spec spec, const char *const *words, size_t word_count, uint32_t seed) { + char *path = new_store(); + CHECK(load(spec, &module_a) == PURE_OK); + static Step history[2000]; + size_t saved = 0, applied = 0; + PureApp memory, stored; + CHECK(pure_app_start(&memory, &module_a, (PureData){0}) == PURE_OK); + CHECK(pure_app_start(&stored, &module_a, pure_tree_data(path)) == PURE_OK); + CHECK(pure_app_defer(&stored, true) && !pure_app_defer(&memory, true)); + unsigned lost = 0, saves = 0; + for (unsigned step = 0; step < 600; ++step) { + Step next = {next_random(&seed) % module_a.inputs, NULL}; + if (module_a.input[next.input].kind == PURE_KIND_TEXT) next.word = words[next_random(&seed) % word_count]; + PureResult a = apply_step(&memory, next), b = apply_step(&stored, next); + CHECK(a == b); + if (a == PURE_OK) history[applied++] = next; + same_states(&memory, &stored); + uint32_t roll = next_random(&seed) % 16; + if (roll < 3) { + CHECK(pure_app_save(&stored) == PURE_OK); + saved = applied; + ++saves; + } else if (roll == 3) { + /* Stop without saving: only saved events come back. */ + pure_app_stop(&stored); + pure_app_stop(&memory); + CHECK(pure_app_start(&stored, &module_a, pure_tree_data(path)) == PURE_OK && pure_app_defer(&stored, true)); + CHECK(pure_app_start(&memory, &module_a, (PureData){0}) == PURE_OK); + for (size_t k = 0; k < saved; ++k) CHECK(apply_step(&memory, history[k]) == PURE_OK); + lost += (unsigned)(applied - saved); + applied = saved; + same_states(&memory, &stored); + } + } + CHECK(saves > 50 && lost > 0); + pure_app_stop(&memory); + pure_app_stop(&stored); + remove_store(path); +} + +/* A burst of events is saved with one commit: when that commit fails every + * event of the burst is undone, and a process that dies while saving comes + * back with all of the burst or none of it. */ +static void deferred_bursts(void) { + char *path = new_store(); + Spec spec = {crash_nums, 1, crash_inputs, 1, NULL, 0, crash_handlers, 1, crash_code, LEN(crash_code), NULL, 0, + crash_states, 4, crash_constants, 2, NULL, 0, crash_records, 1, NULL, 0}; + CHECK(load(spec, &module_a) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && pure_app_defer(&app, true)); + for (int i = 0; i < 5; ++i) CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); + CHECK(consistent(&app) == 5 && app.events == 5); + for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; fail_stage += 2) { + CHECK(pure_app_save(&app) == PURE_STORE_ERROR && consistent(&app) == 0 && app.events == 0); + for (int i = 0; i < 5; ++i) CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); + } + fail_stage = -1; + CHECK(pure_app_save(&app) == PURE_OK && consistent(&app) == 5); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == 5); + pure_app_stop(&app); + unsigned expected = 5; + for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + PureApp doomed; + if (pure_app_start(&doomed, &module_a, pure_tree_data(path)) != PURE_OK || !pure_app_defer(&doomed, true)) _exit(1); + for (int i = 0; i < 7; ++i) if (pure_app_input(&doomed, 0, PURE_EDGE_DOWN) != PURE_OK) _exit(2); + crash = 1; + fail_stage = stage; + pure_app_save(&doomed); + _exit(0); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); + if (stage == PURE_BEFORE_DIR_SYNC) expected += 7; + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == expected); + pure_app_stop(&app); + } + remove_store(path); +} + +/* Messages in a burst: documents of different keys commit together with the + * states, each moving one revision; a second message for a key saves the + * burst first, and so does a full burst. */ +static void deferred_messages(void) { + char *path = new_store(); + CHECK(load(editor_spec, &module_a) == PURE_OK); + PureApp app; + unsigned char buffer[64]; + char key[8]; + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && pure_app_defer(&app, true)); + for (int i = 0; i < 3; ++i) { + snprintf(key, sizeof(key), "k%d", i); + CHECK(pure_app_message(&app, 1, key, 0, edit_payload(buffer, 0, 0, "ab")) == PURE_OK && app.revision == 1); + } + CHECK(app.values[0] == 3 && app.pending_count == 3 && app.open); + /* The same key again: the burst is saved first, then this edit joins a new one. */ + CHECK(pure_app_message(&app, 1, "k0", 1, edit_payload(buffer, 2, 0, "c")) == PURE_OK && app.revision == 2); + CHECK(app.pending_count == 1 && app.values[0] == 4); + CHECK(pure_app_message(&app, 1, "k1", 0, edit_payload(buffer, 0, 0, "x")) == PURE_CONFLICT); + CHECK(pure_app_save(&app) == PURE_OK && !app.open && !app.pending_count); + for (int i = 0; i < PURE_APP_PENDING + 1; ++i) { + snprintf(key, sizeof(key), "m%d", i); + CHECK(pure_app_message(&app, 1, key, 0, edit_payload(buffer, 0, 0, "y")) == PURE_OK); + CHECK(app.pending_count == (i < PURE_APP_PENDING ? (unsigned)i + 1 : 1)); /* full: saved on the way */ + } + pure_app_stop(&app); /* the last edit was never saved */ + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 4 + PURE_APP_PENDING); + CHECK(pure_app_message(&app, 0, "k0", -1, (PureSpan){0}) == PURE_OK && app.revision == 2 && replied(&app, "abc")); + snprintf(key, sizeof(key), "m%d", PURE_APP_PENDING); + CHECK(pure_app_message(&app, 0, key, -1, (PureSpan){0}) == PURE_NOT_FOUND); + snprintf(key, sizeof(key), "m%d", PURE_APP_PENDING - 1); + CHECK(pure_app_message(&app, 0, key, -1, (PureSpan){0}) == PURE_OK && replied(&app, "y")); + pure_app_stop(&app); + remove_store(path); +} + +static void deferred(void) { + static const char *const list_words[] = {"a", "\xea\xb0\x80", "", "\n", "\xf0\x9f\x98\x80"}; + static const char *const text_words[] = {"a", "\xea\xb0\x80", "\n", "\xf0\x9f\x98\x80", "xy\nz", ""}; + deferred_equivalence(list_spec, list_words, LEN(list_words), 0x2545f491u); + deferred_equivalence(text_spec, text_words, LEN(text_words), 0x1b873593u); + deferred_bursts(); + deferred_messages(); +} + +/* Text inserted at a cursor goes before it; at a mark (a cursor that stays), + * after it. Everywhere else, and when text is removed, both move alike. */ +static void marks(void) { + enum { MARK = 2 | PURE_CURSOR_STAYS << 8 }; + static const T textual[] = {{"note", 1, 0}, {"at", 2, 0}, {"mark", MARK, 0}}; + static const char *const constants[] = {"ab"}; + static const I inputs[] = {{"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("keep", "KeyK"), KEY("back", "Backspace")}; + static const O code[] = { + {CUL, 1, 0}, {TI, 0, 0}, {TINS, 0, 0}, /* 0 typed: at the cursor */ + {CUL, 1, 0}, {CS, 2, 0}, /* 3 keep: mark = at */ + {CUL, 1, 0}, {C, 0, -1}, {CM, 0, 0}, {CUL, 1, 0}, {TREM, 0, 0}, /* 5 back */ + }; + static const H handlers[] = {{1, 0, 3}, {3, 3, 2}, {5, 5, 5}}; + Spec spec = {NULL, 0, inputs, LEN(inputs), NULL, 0, handlers, LEN(handlers), code, LEN(code), + NULL, 0, textual, 3, constants, 1, NULL, 0, NULL, 0, NULL, 0}; + if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } + CHECK(module_a.state_flags[2] == PURE_CURSOR_STAYS && !module_a.state_flags[1]); + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"x", 1}) == PURE_OK); + CHECK(!strcmp(state_text(&app, 0), "xab") && app.values[1] == 1 && app.values[2] == 0); /* the mark stayed */ + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 1); + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"yz", 2}) == PURE_OK); + CHECK(!strcmp(state_text(&app, 0), "xyzab") && app.values[1] == 3 && app.values[2] == 1); + CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 2 && app.values[2] == 1); + CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 1 && app.values[2] == 1); /* gathered */ + pure_app_stop(&app); + /* A flag only on a cursor, and only this one. */ + static const T wrong_flag[] = {{"note", 1, 0}, {"at", 2 | 2 << 8, 0}, {"mark", MARK, 0}}; + static const T flagged_text[] = {{"note", 1 | PURE_CURSOR_STAYS << 8, 0}, {"at", 2, 0}, {"mark", 2, 0}}; + Spec bad = spec; + bad.t = wrong_flag; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); + bad.t = flagged_text; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); +} + +/* Folds: a block run once per item of a list, as a column of it, gives a + * count, a sum, a maximum or a minimum; an empty maximum has no value. */ +enum { FOLD = PURE_OP_FOLD }; +#define FOLDING(kind, list, length) ((uint32_t)(kind) | (uint32_t)(list) << 8 | (uint32_t)(length) << 16) +static const RF fold_fields[] = {{"title", 1, 0}, {"done", 0, 0}, {"points", 0, 0}}; +static const R fold_records[] = {{"task", fold_fields, 3}}; +static const T fold_states[] = {{"tasks", 3, 0}, {"at", 4, 0}}; +static const char *const fold_constants[] = {""}; +static const I fold_inputs[] = {{"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("toggle", "KeyT")}; +enum { F_TITLE = 0, F_DONE = 1u << 16, F_POINTS = 2u << 16 }; +static const O fold_code[] = { + {TI, 0, 0}, {C, 0, 0}, {TI, 0, 0}, {CHR, 0, 0}, {LA, 0, 0}, {XE, 0, 0}, {XS, 1, 0}, /* 0 typed */ + {XL, 1, 0}, {XL, 1, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, {IS, F_DONE, 0}, /* 7 toggle */ + {FOLD, FOLDING(PURE_FOLD_COUNT, 0, 3), 0}, {ROW, 0, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, /* 12 open */ + {FOLD, FOLDING(PURE_FOLD_SUM, 0, 3), 0}, {ROW, 0, 0}, {IG, F_POINTS, 0}, {C, 0, 1}, /* 16 points */ + {FOLD, FOLDING(PURE_FOLD_MAX, 0, 4), 0}, {ROW, 0, 0}, {IG, F_TITLE, 0}, {CHR, 0, 0}, /* 20 longest */ + {C, 0, 1}, + {FOLD, FOLDING(PURE_FOLD_SUM, 0, 4), 0}, {ROW, 0, 0}, {IG, F_POINTS, 0}, {ROW, 0, 0}, /* 25 done_points */ + {IG, F_DONE, 0}, + {FOLD, FOLDING(PURE_FOLD_MIN, 0, 5), 0}, {ROW, 0, 0}, {IG, F_POINTS, 0}, {ROW, 0, 0}, /* 30 fewest_open */ + {IG, F_DONE, 0}, {NOT, 0, 0}, + {C, 0, 10}, {FOLD, FOLDING(PURE_FOLD_COUNT, 0, 1), 0}, {C, 0, 1}, {ADD, 0, 0}, /* 36 count_plus */ + {FOLD, FOLDING(PURE_FOLD_FIRST, 0, 3), 0}, {ROW, 0, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, /* 40 first_open */ + {XN, 0, 0}, +}; +static const H fold_handlers[] = {{1, 0, 7}, {3, 7, 5}}; +static const V fold_views[] = {{"open", 12, 4}, {"points", 16, 4}, {"longest", 20, 5}, {"done_points", 25, 5}, + {"fewest_open", 30, 6}, {"count_plus", 36, 4}, {"first_open", 40, 5}}; +static Spec fold_spec(const O *code, size_t count, const V *views, size_t nviews, const H *handlers, size_t nhandlers) { + return (Spec){NULL, 0, fold_inputs, LEN(fold_inputs), views, nviews, handlers, nhandlers, code, count, NULL, 0, + fold_states, 2, fold_constants, 1, NULL, 0, fold_records, 1, NULL, 0}; +} + +static void folds(void) { + CHECK(load(fold_spec(fold_code, LEN(fold_code), fold_views, LEN(fold_views), fold_handlers, 2), &module_a) == PURE_OK); + PureApp app; + double value; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + CHECK(number_view(&app, 0) == 0 && number_view(&app, 1) == 0 && !pure_app_view(&app, 2, &value)); + CHECK(number_view(&app, 3) == 0 && !pure_app_view(&app, 4, &value) && number_view(&app, 5) == 10); + CHECK(!pure_app_view(&app, 6, &value)); /* no first open task yet */ + const char *const words[] = {"a", "bbb", "\xea\xb0\x80\xea\xb0\x80"}; /* one, three and two characters */ + for (size_t i = 0; i < 3; ++i) + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)words[i], strlen(words[i])}) == PURE_OK); + CHECK(number_view(&app, 0) == 3 && number_view(&app, 1) == 6 && number_view(&app, 2) == 3); + CHECK(number_view(&app, 3) == 0 && number_view(&app, 4) == 1 && number_view(&app, 5) == 13); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); /* the last task is done */ + CHECK(number_view(&app, 0) == 2 && number_view(&app, 3) == 2 && number_view(&app, 4) == 1); + CHECK(number_view(&app, 6) == 0); /* the first open one: an index */ + CHECK(pure_app_point(&app, 1, 0) == PURE_OK && pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); + CHECK(number_view(&app, 6) == 1); + CHECK(pure_app_point(&app, 1, 1) == PURE_OK && pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); + CHECK(!pure_app_view(&app, 6, &value)); + CHECK(number_view(&app, 0) == 0 && number_view(&app, 3) == 6 && !pure_app_view(&app, 4, &value)); /* none open */ + pure_app_stop(&app); + + /* What a fold may not be. */ + static const O nested[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 0, 2), 0}, {FOLD, FOLDING(PURE_FOLD_COUNT, 0, 1), 0}, {C, 0, 1}}; + static const O into[] = {{C, 0, 1}, {JZ, 3, 0}, {FOLD, FOLDING(PURE_FOLD_COUNT, 0, 1), 0}, {C, 0, 1}}; + static const O effect[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 0, 3), 0}, {ROW, 0, 0}, {XS, 1, 0}, {C, 0, 1}, + {PURE_OP_REQUIRE, 0, 0}}; + static const O short_sum[] = {{FOLD, FOLDING(PURE_FOLD_SUM, 0, 1), 0}, {C, 0, 1}}; + static const O outside[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 0, 5), 0}, {C, 0, 1}}; + static const O not_list[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 1, 1), 0}, {C, 0, 1}}; + static const struct { const O *code; size_t count, length; bool handler; const char *reason; } refused[] = { + {nested, LEN(nested), 3, false, "fold inside a fold"}, + {into, LEN(into), 4, false, "branch into a fold"}, + {effect, LEN(effect), 5, true, "fold with an effect"}, + {short_sum, LEN(short_sum), 2, false, "fold block must leave its numbers"}, + {outside, LEN(outside), 2, false, "fold block outside its block"}, + {not_list, LEN(not_list), 2, false, "no such state"}, + }; + for (size_t i = 0; i < LEN(refused); ++i) { + V view = {"v", 0, (uint32_t)refused[i].length}; + H handler = {3, 0, (uint32_t)refused[i].length}; + Spec spec = refused[i].handler ? fold_spec(refused[i].code, refused[i].count, NULL, 0, &handler, 1) + : fold_spec(refused[i].code, refused[i].count, &view, 1, NULL, 0); + if (load(spec, &module_b) != PURE_INVALID || !strstr(error, refused[i].reason)) { + fprintf(stderr, "fold case %zu: expected '%s', got '%s'\n", i, refused[i].reason, error); + abort(); + } + } +} + +/* A filtered table: one row per item its filter keeps; its columns read the + * item (ROW is the item, not the row), and hosts turn rows into items and + * back. The rows follow every change. */ +static const O filter_code[] = { + {TI, 0, 0}, {C, 0, 0}, {TI, 0, 0}, {CHR, 0, 0}, {LA, 0, 0}, {XE, 0, 0}, {XS, 1, 0}, /* 0 typed */ + {XL, 1, 0}, {XL, 1, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, {IS, F_DONE, 0}, /* 7 toggle */ + {ROW, 0, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, /* 12 filter: open */ + {ROW, 0, 0}, {IG, F_TITLE, 0}, /* 15 title */ + {ROW, 0, 0}, {XN, 0, 0}, /* 17 pos */ +}; +enum { FILTERED = 2 | 1 << 8 }; /* a table over state 0, filtered */ +static const W filter_views[] = {{"open", FILTERED, 0, 12, 3}, {"title", 1, 1, 15, 2}, {"pos", 0, 1, 17, 2}}; + +static Spec filter_spec(const O *code, size_t count, const W *views, size_t nviews) { + return (Spec){NULL, 0, fold_inputs, LEN(fold_inputs), NULL, 0, fold_handlers, 2, code, count, NULL, 0, + fold_states, 2, fold_constants, 1, NULL, 0, fold_records, 1, views, nviews}; +} + +static const char *cell_title(PureApp *app, uint32_t column, size_t row) { + static Text text; + text = (Text){{0}, 0}; + CHECK(pure_app_cell_text(app, column, row, text_span, &text) == PURE_OK && text.size < sizeof(text.text)); + text.text[text.size] = 0; + return text.text; +} + +static void filters(void) { + CHECK(load(filter_spec(filter_code, LEN(filter_code), filter_views, 3), &module_a) == PURE_OK); + PureApp app; + double value; + size_t at; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK && pure_app_rows(&app, 0) == 0); + for (const char *word = "abc"; *word; ++word) + CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)word, 1}) == PURE_OK); + CHECK(pure_app_rows(&app, 0) == 3 && !strcmp(cell_title(&app, 1, 2), "c") && pure_app_cell(&app, 2, 2, &value) && value == 2); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); /* c is done: it leaves */ + CHECK(pure_app_rows(&app, 0) == 2 && !pure_app_cell(&app, 2, 2, &value) && !pure_app_item_row(&app, 0, 2, &at)); + CHECK(pure_app_point(&app, 1, 0) == PURE_OK && pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); /* a too */ + CHECK(pure_app_rows(&app, 0) == 1 && !strcmp(cell_title(&app, 1, 0), "b")); + CHECK(pure_app_cell(&app, 2, 0, &value) && value == 1); /* the item's position, not the row's */ + CHECK(pure_app_row_item(&app, 0, 0, &at) && at == 1 && !pure_app_row_item(&app, 0, 1, &at)); + CHECK(pure_app_item_row(&app, 0, 1, &at) && at == 0 && !pure_app_item_row(&app, 0, 0, &at)); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); /* a comes back, before b */ + CHECK(pure_app_rows(&app, 0) == 2 && !strcmp(cell_title(&app, 1, 0), "a") && pure_app_item_row(&app, 0, 1, &at) && at == 1); + CHECK(pure_app_replace(&app, &module_a) == PURE_OK && pure_app_rows(&app, 0) == 2); + pure_app_stop(&app); + + /* What a filter may not be. */ + static const O effect[] = {{ROW, 0, 0}, {XS, 1, 0}, {C, 0, 1}}; + static const O text_filter[] = {{ROW, 0, 0}, {IG, F_TITLE, 0}}; + static const O folded[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 0, 1), 0}, {C, 0, 1}}; + static const W filtered_by[] = {{"open", FILTERED, 0, 0, 0}}, open3[] = {{"open", FILTERED, 0, 0, 3}}, + open2[] = {{"open", FILTERED, 0, 0, 2}}; + static const W not_a_list[] = {{"open", 2 | 2 << 8, 0, 0, 3}}; + static const W flagged_column[] = {{"open", FILTERED, 0, 0, 3}, {"c", 0 | 1 << 8, 1, 0, 3}}; + static const struct { const O *code; size_t count; const W *views; size_t nviews; const char *reason; } refused[] = { + {effect, LEN(effect), open3, 1, "filter with an effect"}, + {text_filter, LEN(text_filter), open2, 1, "filter must leave one number"}, + {folded, LEN(folded), open2, 1, "fold inside a filter"}, + {effect, LEN(effect), filtered_by, 1, "table list"}, /* a filtered table needs its block */ + {effect, LEN(effect), not_a_list, 1, "table list"}, /* state 1 is an index */ + {effect, LEN(effect), flagged_column, 2, "view type"}, /* only a table has a list byte */ + }; + for (size_t i = 0; i < LEN(refused); ++i) + if (load(filter_spec(refused[i].code, refused[i].count, refused[i].views, refused[i].nviews), &module_b) != PURE_INVALID || + !strstr(error, refused[i].reason)) { + fprintf(stderr, "filter case %zu: expected '%s', got '%s'\n", i, refused[i].reason, error); + abort(); + } +} + +/* Damaged fold and filter modules: whatever loads runs safely. */ +static void fold_mutations(void) { + static unsigned char valid[65536], bytes[65536]; + size_t size = build(valid, fold_spec(fold_code, LEN(fold_code), fold_views, LEN(fold_views), fold_handlers, 2)); + for (unsigned kind = 0; kind < 2; ++kind) { + if (kind) size = build(valid, filter_spec(filter_code, LEN(filter_code), filter_views, 3)); + uint32_t state = kind ? 0x165667b1u : 0x27d4eb2fu; + unsigned accepted = 0; + for (unsigned round = 0; round < 20000; ++round) { + memcpy(bytes, valid, size); + for (unsigned f = 0, flips = 1 + next_random(&state) % 3; f < flips; ++f) + bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); + if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; + ++accepted; + PureApp app; + if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; + for (unsigned e = 0; e < 16; ++e) { + uint32_t input = next_random(&state) % 2; + if (module_b.input[input].kind == PURE_KIND_TEXT) pure_app_text(&app, input, (PureSpan){(const unsigned char *)"ab", 2}); + else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); + for (uint32_t v = 0; v < module_b.views; ++v) { + double value; + size_t item; + if (module_b.view_type[v] == PURE_VIEW_TABLE) { + size_t rows = pure_app_rows(&app, v); + for (size_t row = 0; row < rows + 1; ++row) pure_app_row_item(&app, v, row, &item); + } else if (module_b.view_table[v]) { + for (size_t row = 0; row < 4; ++row) + if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_cell_text(&app, v, row, ignore_span, NULL); + else pure_app_cell(&app, v, row, &value); + } else if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); + else pure_app_view(&app, v, &value); + } + } + pure_app_stop(&app); + } + printf("{\"%s_module_mutations\":20000,\"accepted_after_damage\":%u}\n", kind ? "filter" : "fold", accepted); + } +} + +/* A host points at an item: the index a view shows as its position moves + * there, as an event with no handler, saved like a key event; deferred, it + * joins the burst and a failed save undoes it with the rest. */ +static void pointing(void) { + CHECK(load(list_spec, &module_a) == PURE_OK); + CHECK(pure_module_position(&module_a, 1) == 2); /* position = at.position */ + CHECK(pure_module_position(&module_a, 0) == -1 && pure_module_position(&module_a, 2) == -1 && + pure_module_position(&module_a, 3) == -1 && pure_module_position(&module_a, 99) == -1); + char path[] = "/tmp/pure-point-XXXXXX"; + CHECK(mkdtemp(path) && pure_store_init(path) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); + CHECK(pure_app_point(&app, 2, 0) == PURE_NOT_FOUND); /* nothing to point at */ + for (int i = 0; i < 3; ++i) CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"a", 1}) == PURE_OK); + uint64_t events = app.events; + CHECK(app.values[2] == 2 && pure_app_point(&app, 2, 0) == PURE_OK && app.values[2] == 0 && app.events == events + 1); + CHECK(pure_app_point(&app, 2, 3) == PURE_NOT_FOUND && app.values[2] == 0); + CHECK(pure_app_point(&app, 0, 1) == PURE_INVALID && pure_app_point(&app, 1, 1) == PURE_INVALID); /* a number, a list */ + CHECK(pure_app_point(&app, 2, 1) == PURE_OK); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[2] == 1); + CHECK(pure_app_defer(&app, true) && pure_app_point(&app, 2, 2) == PURE_OK && app.open && app.values[2] == 2); + CHECK(pure_app_save(&app) == PURE_OK); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[2] == 2); + pure_app_stop(&app); + remove_store(path); +} + +/* Keyed lists: items in key order (texts by bytes, numbers by value), each + * key once; found by key; a list that becomes keyed is put in order with + * its indices following their items, or refused when two items share a key. */ +enum { FIND = PURE_OP_LIST_FIND, HAS = PURE_OP_LIST_HAS, TCMP = PURE_OP_TEXT_COMPARE, + PEOPLE = 1, K_AT = 2, SCORES = 3, K_NAME = 0, K_VISITS = 1u << 16 }; +static const S keyed_nums[] = {{"n", 0}}; +static const T keyed_states[] = {{"people", 3 | 1 << 8, 0}, {"at", 4, 1}, {"scores", 3 | 1 << 8, 1}}; +static const T unkeyed_states[] = {{"people", 3, 0}, {"at", 4, 1}, {"scores", 3, 1}}; +static const RF person_fields[] = {{"name", 1, 0}, {"visits", 0, 0}}; +static const RF score_fields[] = {{"score", 0, 0}}; +static const R keyed_records[] = {{"person", person_fields, 2}, {"score", score_fields, 1}}; +static const char *const keyed_constants[] = {"", "bob", "m"}; +static const I keyed_inputs[] = { + {"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, {"seek", NULL, PURE_KIND_TEXT, false, NULL, 0}, + {"has", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("drop", "Delete"), KEY("bump", "KeyB"), + KEY("two", "KeyN"), KEY("zero", "KeyO"), KEY("minus_zero", "KeyM"), {"twice", NULL, PURE_KIND_TEXT, false, NULL, 0}, +}; +static const O keyed_code[] = { + {TI, 0, 0}, {C, 0, 0}, {LA, PEOPLE, 0}, {TI, 0, 0}, {FIND, PEOPLE, 0}, {XS, K_AT, 0}, /* 0 typed: add, point at it */ + {TI, 0, 0}, {FIND, PEOPLE, 0}, {XS, K_AT, 0}, /* 6 seek */ + {TI, 0, 0}, {HAS, PEOPLE, 0}, {ST, 0, 0}, /* 9 has */ + {XL, K_AT, 0}, {LR, PEOPLE, 0}, /* 12 drop */ + {XL, K_AT, 0}, {XL, K_AT, 0}, {IG, PEOPLE | K_VISITS, 0}, {C, 0, 1}, {ADD, 0, 0}, /* 14 bump */ + {IS, PEOPLE | K_VISITS, 0}, + {C, 0, 2}, {LA, SCORES, 0}, {C, 0, 0}, {LA, SCORES, 0}, {C, 0, -0.0}, {LA, SCORES, 0}, /* 20 two, zero, minus_zero */ + {TI, 0, 0}, {C, 0, 0}, {LA, PEOPLE, 0}, {TI, 0, 0}, {C, 0, 0}, {LA, PEOPLE, 0}, /* 26 twice: always exists */ + {LC, PEOPLE, 0}, /* 32 view count */ + {XL, K_AT, 0}, {XN, 0, 0}, /* 33 view position */ + {XL, K_AT, 0}, {IG, PEOPLE | K_NAME, 0}, {TK, 1, 0}, {TCMP, PURE_EQUAL, 0}, /* 35 view is_bob */ + {XL, K_AT, 0}, {IG, PEOPLE | K_NAME, 0}, {TK, 2, 0}, {TCMP, PURE_LESS, 0}, /* 39 view before_m */ + {LC, SCORES, 0}, /* 43 view scores */ + {XL, K_AT, 0}, {IG, PEOPLE | K_NAME, 0}, /* 44 view current */ +}; +static const H keyed_handlers[] = {{1, 0, 6}, {3, 6, 3}, {5, 9, 3}, {7, 12, 2}, {9, 14, 6}, {11, 20, 2}, {13, 22, 2}, + {15, 24, 2}, {17, 26, 6}}; +static const V keyed_views[] = {{"count", 32, 1}, {"position", 33, 2}, {"is_bob", 35, 4}, {"before_m", 39, 4}, + {"scores_count", 43, 1}}; +static const V keyed_text_views[] = {{"current", 44, 2}}; +/* Unkeyed, the same states: typed appends and points at the new last item. */ +static const O unkeyed_code[] = {{TI, 0, 0}, {C, 0, 0}, {LA, PEOPLE, 0}, {XE, PEOPLE, 0}, {XS, K_AT, 0}}; +static const H unkeyed_handlers[] = {{1, 0, 5}}; + +static Spec keyed_spec(const T *states, const O *code, size_t count, const H *handlers, size_t nhandlers) { + bool views = code == keyed_code; + return (Spec){keyed_nums, 1, keyed_inputs, LEN(keyed_inputs), keyed_views, views ? LEN(keyed_views) : 0, handlers, + nhandlers, code, count, NULL, 0, states, LEN(keyed_states), keyed_constants, LEN(keyed_constants), + keyed_text_views, views ? 1 : 0, keyed_records, 2, NULL, 0}; +} + +static PureResult type_text(PureApp *app, uint32_t input, const char *text) { + return pure_app_text(app, input, (PureSpan){(const unsigned char *)text, strlen(text)}); +} + +/* A list's first field (text) of every item, comma separated. */ +static const char *names_of(const PureApp *app, uint32_t list) { + static char out[512]; + size_t used = 0; + for (size_t at = 0; at < pure_list_count(app->lists[list]); ++at) { + PureCell cell = pure_list_get(app->lists[list], at, 0); + CHECK(used + cell.size + 2 < sizeof(out)); + if (at) out[used++] = ','; + if (cell.size) memcpy(out + used, cell.text, cell.size); + used += cell.size; + } + out[used] = 0; + return out; +} + +static void keyed(void) { + CHECK(load(keyed_spec(keyed_states, keyed_code, LEN(keyed_code), keyed_handlers, LEN(keyed_handlers)), &module_a) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + for (const char *const *word = (const char *const[]){"m", "b", "z", "a", NULL}; *word; ++word) + CHECK(type_text(&app, 0, *word) == PURE_OK); + CHECK(!strcmp(names_of(&app, PEOPLE), "a,b,m,z") && app.values[K_AT] == 0 && !strcmp(view_text(&app, 5), "a")); + /* A key once: the add fails, and so does an event whose second add repeats its first. */ + CHECK(type_text(&app, 0, "b") == PURE_CONFLICT && app.fault == PURE_FAULT_EXISTS); + CHECK(type_text(&app, 8, "c") == PURE_CONFLICT && !strcmp(names_of(&app, PEOPLE), "a,b,m,z") && app.values[K_AT] == 0); + CHECK(type_text(&app, 1, "z") == PURE_OK && app.values[K_AT] == 3 && !strcmp(view_text(&app, 5), "z")); + CHECK(type_text(&app, 1, "q") == PURE_NOT_FOUND && app.fault == PURE_FAULT_NOT_FOUND && app.values[K_AT] == 3); + CHECK(type_text(&app, 2, "m") == PURE_OK && app.values[0] == 1); + CHECK(type_text(&app, 2, "c") == PURE_OK && app.values[0] == 0); + CHECK(pure_app_input(&app, 4, PURE_EDGE_DOWN) == PURE_OK && pure_list_get(app.lists[PEOPLE], 3, 1).num == 1); + CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && !strcmp(names_of(&app, PEOPLE), "a,b,m")); + CHECK(app.values[K_AT] == 2 && number_view(&app, 2) == 0 && number_view(&app, 3) == 0); /* "m": not bob, not before m */ + CHECK(type_text(&app, 1, "b") == PURE_OK && number_view(&app, 2) == 0 && number_view(&app, 3) == 1); + CHECK(type_text(&app, 0, "bob") == PURE_OK && number_view(&app, 2) == 1 && number_view(&app, 3) == 1); + /* Bytes order texts: a text before the longer ones it begins, UTF-8 after ASCII, the empty text first. */ + CHECK(type_text(&app, 0, "\xea\xb0\x80") == PURE_OK && type_text(&app, 0, "ab") == PURE_OK); + CHECK(type_text(&app, 0, "") == PURE_OK && type_text(&app, 0, "") == PURE_CONFLICT); + CHECK(!strcmp(names_of(&app, PEOPLE), ",a,ab,b,bob,m,\xea\xb0\x80") && app.values[K_AT] == 0); + /* Numbers order by value, and -0 is the key 0. */ + CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && pure_app_input(&app, 6, PURE_EDGE_DOWN) == PURE_OK); + CHECK(pure_app_input(&app, 7, PURE_EDGE_DOWN) == PURE_CONFLICT && number_view(&app, 4) == 2); + CHECK(pure_list_get(app.lists[SCORES], 0, 0).num == 0 && !signbit(pure_list_get(app.lists[SCORES], 0, 0).num) && + pure_list_get(app.lists[SCORES], 1, 0).num == 2); + + /* Replacing: unkeyed, the list stays as it is and takes appends at its end. */ + CHECK(load(keyed_spec(unkeyed_states, unkeyed_code, LEN(unkeyed_code), unkeyed_handlers, 1), &module_b) == PURE_OK); + CHECK(pure_app_replace(&app, &module_b) == PURE_OK && type_text(&app, 0, "c") == PURE_OK); + CHECK(!strcmp(names_of(&app, PEOPLE), ",a,ab,b,bob,m,\xea\xb0\x80,c") && app.values[K_AT] == 7); + CHECK(pure_app_point(&app, K_AT, 6) == PURE_OK); /* at the Korean name */ + /* Keyed again: put in order, the index following its item. */ + CHECK(pure_app_replace(&app, &module_a) == PURE_OK && !strcmp(names_of(&app, PEOPLE), ",a,ab,b,bob,c,m,\xea\xb0\x80")); + CHECK(app.values[K_AT] == 7 && !strcmp(view_text(&app, 5), "\xea\xb0\x80")); + /* Two items with one key cannot become keyed: the app keeps its module and items. */ + CHECK(pure_app_replace(&app, &module_b) == PURE_OK && type_text(&app, 0, "a") == PURE_OK); + CHECK(pure_app_replace(&app, &module_a) == PURE_CONFLICT && app.module == &module_b && pure_list_count(app.lists[PEOPLE]) == 9); + pure_app_stop(&app); + + /* Stored: a list that becomes keyed is stored in order, once. */ + char *path = new_store(); + CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK); + for (const char *const *word = (const char *const[]){"m", "b", "z", NULL}; *word; ++word) + CHECK(type_text(&app, 0, *word) == PURE_OK); + CHECK(pure_app_point(&app, K_AT, 1) == PURE_OK); + pure_app_stop(&app); + for (int round = 0; round < 2; ++round) { + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); + CHECK(!strcmp(names_of(&app, PEOPLE), "b,m,z") && app.values[K_AT] == 0); + pure_app_stop(&app); + } + CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK && type_text(&app, 0, "m") == PURE_OK); + pure_app_stop(&app); + CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_CONFLICT); + CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK && !strcmp(names_of(&app, PEOPLE), "b,m,z,m")); + pure_app_stop(&app); + remove_store(path); + static const char *const words[] = {"a", "b", "ab", "", "\xea\xb0\x80", "zz"}; + for (uint32_t seed = 1; seed <= 3; ++seed) + stored_equivalence(keyed_spec(keyed_states, keyed_code, LEN(keyed_code), keyed_handlers, LEN(keyed_handlers)), + words, LEN(words), seed * 0x9e3779b9u); + + /* What a keyed list and a text comparison may not be. */ + static const O insert[] = {{XF, PEOPLE, 0}, {TK, 0, 0}, {C, 0, 0}, {LI, PEOPLE, 0}}; + static const O set_key[] = {{XL, K_AT, 0}, {TK, 0, 0}, {IS, PEOPLE | K_NAME, 0}}; + static const O find_number[] = {{C, 0, 1}, {FIND, PEOPLE, 0}, {XS, K_AT, 0}}; + static const O find[] = {{TI, 0, 0}, {FIND, PEOPLE, 0}, {XS, K_AT, 0}}; + static const O relation[] = {{TI, 0, 0}, {TI, 0, 0}, {TCMP, PURE_NOT_EQUAL + 1, 0}, {ST, 0, 0}}; + static const O compare_numbers[] = {{C, 0, 1}, {C, 0, 1}, {TCMP, PURE_EQUAL, 0}, {ST, 0, 0}}; + static const T far_key[] = {{"people", 3 | 3 << 8, 0}, {"at", 4, 1}, {"scores", 3, 1}}; + static const T keyed_index[] = {{"people", 3, 0}, {"at", 4 | 1 << 8, 1}, {"scores", 3, 1}}; + static const struct { const T *states; const O *code; size_t count; const char *reason; } refused[] = { + {keyed_states, insert, LEN(insert), "insert by position into a keyed list"}, + {keyed_states, set_key, LEN(set_key), "change of a keyed list's key"}, + {keyed_states, find_number, LEN(find_number), "operand type"}, + {unkeyed_states, find, LEN(find), "no such state"}, + {keyed_states, relation, LEN(relation), "no such state"}, + {keyed_states, compare_numbers, LEN(compare_numbers), "operand type"}, + {far_key, unkeyed_code, LEN(unkeyed_code), "state reference"}, + {keyed_index, unkeyed_code, LEN(unkeyed_code), "state reference"}, + }; + for (size_t i = 0; i < LEN(refused); ++i) { + H handler = {1, 0, (uint32_t)refused[i].count}; + if (load(keyed_spec(refused[i].states, refused[i].code, refused[i].count, &handler, 1), &module_b) != PURE_INVALID || + !strstr(error, refused[i].reason)) { + fprintf(stderr, "keyed case %zu: expected '%s', got '%s'\n", i, refused[i].reason, error); + abort(); + } + } +} + +/* Damaged keyed modules: whatever loads keeps every keyed list in strict key + * order and every index on an item. */ +static void keyed_mutations(void) { + static unsigned char valid[65536], bytes[65536]; + size_t size = build(valid, keyed_spec(keyed_states, keyed_code, LEN(keyed_code), keyed_handlers, LEN(keyed_handlers))); + static const char *const words[] = {"a", "b", "ab", "", "\xea\xb0\x80"}; + uint32_t state = 0x6c8e9cf5u; + unsigned accepted = 0; + for (unsigned round = 0; round < 20000; ++round) { + memcpy(bytes, valid, size); + for (unsigned f = 0, flips = 1 + next_random(&state) % 3; f < flips; ++f) + bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); + if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; + ++accepted; + PureApp app; + if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; + for (unsigned e = 0; e < 24; ++e) { + uint32_t input = next_random(&state) % module_b.inputs; + if (module_b.input[input].kind == PURE_KIND_TEXT) type_text(&app, input, words[next_random(&state) % LEN(words)]); + else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); + for (uint32_t v = 0; v < module_b.views; ++v) { + double value; + if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); + else pure_app_view(&app, v, &value); + } + for (uint32_t s = 0; s < module_b.states; ++s) { + if (module_b.state_type[s] == PURE_STATE_INDEX) { + size_t count = pure_list_count(app.lists[module_b.state_ref[s]]); + CHECK(app.values[s] == floor(app.values[s]) && app.values[s] >= 0 && + app.values[s] <= (count ? (double)(count - 1) : 0)); + } + if (module_b.state_type[s] != PURE_STATE_LIST || !module_b.state_key[s]) continue; + unsigned key = module_b.state_key[s] - 1u; + bool text = module_b.record_field_text[module_b.record[module_b.state_ref[s]].first + key]; + for (size_t at = 1; at < pure_list_count(app.lists[s]); ++at) { + PureCell a = pure_list_get(app.lists[s], at - 1, key), b = pure_list_get(app.lists[s], at, key); + if (!text) { CHECK(a.num < b.num); continue; } + size_t shorter = a.size < b.size ? a.size : b.size; + int order = shorter ? memcmp(a.text, b.text, shorter) : 0; + CHECK(order < 0 || (!order && a.size < b.size)); + } + } + } + pure_app_stop(&app); + } + printf("{\"keyed_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); +} + +/* ---- Snapshots. Everything an app's states and views show, as text (safe + * on any thread): a reader of a snapshot must show what the app showed. */ +typedef struct { char *text; size_t used, size; } Shown; + +static bool show_span(void *context, PureSpan span) { + Shown *s = context; + CHECK(s->used + span.size < s->size); + if (span.size) memcpy(s->text + s->used, span.data, span.size); + s->used += span.size; + return true; +} + +static void show(Shown *s, const char *format, double value) { + int n = snprintf(s->text + s->used, s->size - s->used, format, value); + CHECK(n >= 0 && (size_t)n < s->size - s->used); + s->used += (size_t)n; +} + +static void describe(PureApp *app, Shown *s) { + const PureModule *m = app->module; + s->used = 0; + for (uint32_t i = 0; i < m->states; ++i) { + if (m->state_type[i] == PURE_STATE_TEXT) + CHECK(pure_draft_read(app->texts[i], 0, pure_draft_size(app->texts[i]), show_span, s) == PURE_OK); + else if (m->state_type[i] == PURE_STATE_LIST) { + const PureRecord *r = &m->record[m->state_ref[i]]; + for (size_t at = 0; at < pure_list_count(app->lists[i]); ++at) + for (unsigned f = 0; f < r->count; ++f) { + PureCell cell = pure_list_get(app->lists[i], at, f); + if (m->record_field_text[r->first + f]) show_span(s, (PureSpan){cell.text, cell.size}); + else show(s, "%g", cell.num); + show(s, f + 1 < r->count ? "," : ";", 0); + } + } else show(s, "%g", app->values[i]); + show(s, "|", 0); + } + for (uint32_t v = 0; v < m->views; ++v) { + double value; + if (m->view_table[v] || m->view_type[v] == PURE_VIEW_TABLE) continue; + if (m->view_type[v] == PURE_VIEW_TEXT) { + if (pure_app_view_text(app, v, show_span, s) != PURE_OK) show(s, "!", 0); + } else if (pure_app_view(app, v, &value)) show(s, "%g", value); + else show(s, "!", 0); + show(s, "|", 0); + } + s->text[s->used] = 0; +} + +/* One random event of the keyed app (see keyed), saved as it returns. */ +static void keyed_event(PureApp *app, uint32_t *seed) { + static const char *const names[] = {"a", "b", "ab", "", "\xea\xb0\x80", "zz", "bob", "m", "q", "c"}; + uint32_t input = next_random(seed) % app->module->inputs; + if (app->module->input[input].kind == PURE_KIND_TEXT) type_text(app, input, names[next_random(seed) % LEN(names)]); + else pure_app_input(app, input, PURE_EDGE_DOWN); +} + +/* A snapshot shows the states of the save before it, as they were, however + * the app goes on (items it changes or drops, a module it switches to, even + * after it stops and its store closes); a change not saved yet refuses one, + * and a save that fails is never shown. */ +static size_t snapshot_retired, snapshot_reads; + +static void snapshots_kept(Spec spec, const Spec *other, void (*event)(PureApp *, uint32_t *), unsigned steps) { + enum { KEPT = 8, SIZE = 1 << 16 }; + static char texts[KEPT + 2][SIZE]; + char *path = new_store(); + PureStore *store; + CHECK(load(spec, &module_a) == PURE_OK && pure_store_open(path, &store) == PURE_OK); + CHECK(!other || load(*other, &module_b) == PURE_OK); + PureApp app, reader = {0}; + CHECK(pure_app_start(&app, &module_a, pure_store_data(store)) == PURE_OK); + PureSnapshot *kept[KEPT] = {NULL}; + Shown seen[KEPT], now = {texts[KEPT], 0, SIZE}, shown = {texts[KEPT + 1], 0, SIZE}; + for (unsigned k = 0; k < KEPT; ++k) seen[k] = (Shown){texts[k], 0, SIZE}; + uint32_t seed = 0x51a95u; + for (unsigned step = 0; step < steps; ++step) { + /* Late on, another module (the keyed app unkeyed), and back if it can. */ + if (other && (step == steps - 400 || step == steps - 300)) + CHECK(pure_app_replace(&app, app.module == &module_a ? &module_b : &module_a) != PURE_INVALID); + event(&app, &seed); + snapshot_retired += app.retired.count; + PureSnapshot *made = pure_app_snapshot(&app); + CHECK(made); + describe(&app, &now); + pure_app_read(&reader, made); + describe(&reader, &shown); + CHECK(!strcmp(now.text, shown.text)); + unsigned slot = next_random(&seed) % KEPT; + if (kept[slot]) { + pure_app_read(&reader, kept[slot]); + describe(&reader, &shown); + CHECK(!strcmp(seen[slot].text, shown.text)); + pure_snapshot_release(kept[slot]); + } + kept[slot] = made; + memcpy(seen[slot].text, now.text, now.used + 1); + } + /* Deferred: an unsaved change refuses a snapshot, and a failed save + * leaves the one before it true. */ + CHECK(pure_app_defer(&app, true)); + describe(&app, &now); + PureResult saved = PURE_OK; + for (unsigned tries = 0; tries < 64 && saved == PURE_OK; ++tries) { + event(&app, &seed); + if (!app.open) continue; + CHECK(!pure_app_snapshot(&app)); + fail_stage = PURE_AFTER_HEADER; + saved = pure_app_save(&app); /* OK when the event changed nothing */ + fail_stage = -1; + } + CHECK(saved == PURE_STORE_ERROR); + PureSnapshot *made = pure_app_snapshot(&app); + CHECK(made); + pure_app_read(&reader, made); + describe(&reader, &shown); + CHECK(!strcmp(now.text, shown.text)); + pure_snapshot_release(made); + /* The kept ones outlive the app and its store. */ + pure_app_stop(&app); + pure_store_close(store); + for (unsigned k = 0; k < KEPT; ++k) { + pure_app_read(&reader, kept[k]); + describe(&reader, &shown); + CHECK(!strcmp(seen[k].text, shown.text)); + pure_snapshot_release(kept[k]); + } + pure_app_stop(&reader); + remove_store(path); +} + +/* Readers on two threads take the newest snapshot while the app applies + * events and makes one after each save: each shows what the app showed. */ +typedef struct { + pthread_mutex_t lock; + PureSnapshot *newest; + char *seen; + bool done; + unsigned reads; +} Published; + +static void *read_newest(void *context) { + Published *p = context; + PureApp reader = {0}; + static _Thread_local char a[1 << 16], b[1 << 16]; + Shown expected = {a, 0, sizeof(a)}, shown = {b, 0, sizeof(b)}; + for (bool done = false; !done;) { + pthread_mutex_lock(&p->lock); + PureSnapshot *s = p->newest; + if (s) { + pure_snapshot_hold(s); + strcpy(expected.text, p->seen); + } + done = p->done; + pthread_mutex_unlock(&p->lock); + if (!s) continue; + pure_app_read(&reader, s); + describe(&reader, &shown); + CHECK(!strcmp(expected.text, shown.text)); + pure_snapshot_release(s); + __atomic_add_fetch(&p->reads, 1, __ATOMIC_RELAXED); + } + pure_app_stop(&reader); + return NULL; +} + +static void snapshots_read(Spec spec, void (*event)(PureApp *, uint32_t *), unsigned steps) { + static char seen[1 << 16], now[1 << 16]; + char *path = new_store(); + PureStore *store; + CHECK(load(spec, &module_a) == PURE_OK && pure_store_open(path, &store) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, pure_store_data(store)) == PURE_OK); + Published p = {.lock = PTHREAD_MUTEX_INITIALIZER, .seen = seen}; + pthread_t readers[2]; + for (unsigned r = 0; r < 2; ++r) CHECK(pthread_create(&readers[r], NULL, read_newest, &p) == 0); + Shown shown = {now, 0, sizeof(now)}; + uint32_t seed = 0x7eadu; + for (unsigned step = 0; step < steps; ++step) { + event(&app, &seed); + PureSnapshot *made = pure_app_snapshot(&app); + CHECK(made); + describe(&app, &shown); + pthread_mutex_lock(&p.lock); + PureSnapshot *old = p.newest; + p.newest = made; + memcpy(seen, shown.text, shown.used + 1); + pthread_mutex_unlock(&p.lock); + pure_snapshot_release(old); + } + pthread_mutex_lock(&p.lock); + p.done = true; + pthread_mutex_unlock(&p.lock); + for (unsigned r = 0; r < 2; ++r) CHECK(pthread_join(readers[r], NULL) == 0); + CHECK(p.reads > steps / 4); + snapshot_reads += p.reads; + pure_snapshot_release(p.newest); + pure_app_stop(&app); + pure_store_close(store); + remove_store(path); +} + +static void crash_event(PureApp *app, uint32_t *seed) { + (void)seed; + CHECK(pure_app_input(app, 0, PURE_EDGE_DOWN) == PURE_OK); +} + +static void snapshots(void) { + Spec keyed = keyed_spec(keyed_states, keyed_code, LEN(keyed_code), keyed_handlers, LEN(keyed_handlers)); + Spec crashing = {crash_nums, 1, crash_inputs, 1, NULL, 0, crash_handlers, 1, crash_code, LEN(crash_code), NULL, 0, + crash_states, 4, crash_constants, 2, NULL, 0, crash_records, 1, NULL, 0}; + Spec unkeyed = keyed_spec(unkeyed_states, unkeyed_code, LEN(unkeyed_code), unkeyed_handlers, 1); + snapshots_kept(keyed, &unkeyed, keyed_event, 3000); + snapshots_kept(crashing, NULL, crash_event, 300); + snapshots_read(keyed, keyed_event, 3000); + snapshots_read(crashing, crash_event, 300); + /* Without stored states (texts only in memory) there is no snapshot. */ + CHECK(load(crashing, &module_a) == PURE_OK); + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK && !pure_app_snapshot(&app)); + pure_app_stop(&app); + CHECK(snapshot_retired > 100); + printf("{\"snapshots\":6600,\"items_retired\":%zu,\"reads_on_threads\":%zu}\n", snapshot_retired, snapshot_reads); +} + +/* A message of a text, a number, bytes and a u32, in that order: the prefix + * holds the text's length, the number and the u32; the text's bytes come + * first after it and the bytes (the last variable field) take the rest. A + * length past the payload, a text that is not UTF-8 or a number that is not + * finite fails the message. */ +enum { TSET = PURE_OP_TEXT_SET }; +static void message_fields(void) { + static const S nums[] = {{"n", 0}}; + static const T texts[] = {{"title", 1, 0}}; + static const char *const constants[] = {""}; + static const F fields[] = {{"title", PURE_FIELD_TEXT}, {"points", PURE_FIELD_NUMBER}, {"note", PURE_FIELD_BYTES}, + {"count", PURE_FIELD_U32}}; + static const I inputs[] = {{"post", NULL, PURE_KIND_MESSAGE, false, fields, 4}}; + static const O code[] = {{FIELD, 1, 0}, {FIELD, 3, 0}, {ADD, 0, 0}, {ST, 0, 0}, {FIELD, 0, 0}, {TSET, 1, 0}, + {FIELD, 2, 0}, {REPLY, 0, 0}}; + static const H handlers[] = {{1, 0, 8}}; + Spec spec = {nums, 1, inputs, 1, NULL, 0, handlers, 1, code, LEN(code), NULL, 0, texts, 1, constants, 1, NULL, 0, + NULL, 0, NULL, 0}; + if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } + CHECK(module_a.input[0].prefix == 16 && module_a.field[0].offset == 0 && module_a.field[1].offset == 4 && + module_a.field[2].offset == PURE_FIELD_REST && module_a.field[3].offset == 12); + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + unsigned char payload[64]; + /* title "가나" (6 bytes), points 2.5, count 3, then the title's bytes and note "xyz". */ + size_t size = 0; + uint32_t title_size = 6, count = 3; + double points = 2.5; + memcpy(payload, &title_size, 4); + memcpy(payload + 4, &points, 8); + memcpy(payload + 12, &count, 4); + memcpy(payload + 16, "\xea\xb0\x80\xeb\x82\x98" "xyz", 9); + size = 25; + CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, size}) == PURE_OK); + CHECK(app.values[0] == 5.5 && !strcmp(state_text(&app, 1), "\xea\xb0\x80\xeb\x82\x98") && replied(&app, "xyz")); + pure_app_finish(&app); + /* Empty variable fields. */ + title_size = 0; + memcpy(payload, &title_size, 4); + CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 16}) == PURE_OK && !*state_text(&app, 1) && replied(&app, "")); + pure_app_finish(&app); + /* A title longer than what is there, a title that is not UTF-8, a number that is not finite, a short prefix. */ + title_size = 10; + memcpy(payload, &title_size, 4); + CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 25}) == PURE_LIMIT && app.fault == PURE_FAULT_LIMIT); + title_size = 2; + memcpy(payload, &title_size, 4); + memcpy(payload + 16, "\xff\xfe", 2); + CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 18}) == PURE_INVALID && app.fault == PURE_FAULT_ENCODING); + title_size = 0; + points = NAN; + memcpy(payload, &title_size, 4); + memcpy(payload + 4, &points, 8); + CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 16}) == PURE_INVALID && app.fault == PURE_FAULT_NON_FINITE); + CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 15}) == PURE_LIMIT); + CHECK(app.values[0] == 5.5 && !*state_text(&app, 1)); /* failed messages changed nothing */ + pure_app_stop(&app); + /* A text field is text and a bytes field bytes. */ + static const O text_as_bytes[] = {{FIELD, 0, 0}, {LEN_, 0, 0}, {ST, 0, 0}}; + static const O bytes_as_text[] = {{FIELD, 2, 0}, {TSET, 1, 0}}; + static const H three[] = {{1, 0, 3}}, two[] = {{1, 0, 2}}; + Spec bad = spec; + bad.o = text_as_bytes; bad.no = 3; bad.h = three; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); + bad.o = bytes_as_text; bad.no = 2; bad.h = two; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); + + /* Placed by the host: the variable fields wherever they are in the + * payload (here a JSON body after the prefix), in any order; the + * prefix's lengths are not read. */ + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + static const char body[] = "{\"note\":\"xyz\",\"title\":\"\xea\xb0\x80\xeb\x82\x98\"}"; + points = 2.5; + title_size = 999; + memcpy(payload, &title_size, 4); + memcpy(payload + 4, &points, 8); + memcpy(payload + 12, &count, 4); + memcpy(payload + 16, body, sizeof(body) - 1); + size = 16 + sizeof(body) - 1; + uint32_t placed[8] = {16 + 23, 6, 0, 0, 16 + 9, 3, 0, 0}; + CHECK(pure_app_message_fields(&app, 0, NULL, -1, (PureSpan){payload, size}, placed) == PURE_OK); + CHECK(app.values[0] == 5.5 && !strcmp(state_text(&app, 1), "\xea\xb0\x80\xeb\x82\x98") && replied(&app, "xyz")); + pure_app_finish(&app); + /* A field that starts or ends past the payload, or a title that is not UTF-8 there. */ + placed[0] = (uint32_t)size + 1; + CHECK(pure_app_message_fields(&app, 0, NULL, -1, (PureSpan){payload, size}, placed) == PURE_LIMIT); + placed[0] = (uint32_t)size - 2; + placed[1] = 3; + CHECK(pure_app_message_fields(&app, 0, NULL, -1, (PureSpan){payload, size}, placed) == PURE_LIMIT); + placed[0] = 16 + 24; + placed[1] = 2; + CHECK(pure_app_message_fields(&app, 0, NULL, -1, (PureSpan){payload, size}, placed) == PURE_INVALID && + app.fault == PURE_FAULT_ENCODING); + CHECK(app.values[0] == 5.5); + pure_app_stop(&app); + + /* Fields in groups: a name is a path of identifiers (user.name), and + * never both a field and a group. */ + static const F grouped[] = {{"user.name", PURE_FIELD_TEXT}, {"user.age", PURE_FIELD_U32}, {"plan", PURE_FIELD_TEXT}}; + static const I grouped_inputs[] = {{"post", NULL, PURE_KIND_MESSAGE, false, grouped, 3}}; + static const O grouped_code[] = {{FIELD, 1, 0}, {ST, 0, 0}, {FIELD, 0, 0}, {TSET, 1, 0}}; + static const H grouped_handlers[] = {{1, 0, 4}}; + Spec groups = spec; + groups.in = grouped_inputs; groups.o = grouped_code; groups.no = LEN(grouped_code); groups.h = grouped_handlers; + CHECK(load(groups, &module_b) == PURE_OK && !strcmp(module_b.field[0].name, "user.name")); + static const char *const refused_paths[] = {"user..name", ".user", "user.", "user.1st", "user name"}; + for (size_t i = 0; i < LEN(refused_paths); ++i) { + F renamed[3] = {grouped[0], grouped[1], grouped[2]}; + renamed[0].name = refused_paths[i]; + I renamed_inputs[1] = {{"post", NULL, PURE_KIND_MESSAGE, false, renamed, 3}}; + groups.in = renamed_inputs; + CHECK(load(groups, &module_b) == PURE_INVALID && strstr(error, "message field")); + } + F both[3] = {{"user", PURE_FIELD_TEXT}, grouped[1], grouped[2]}; + I both_inputs[1] = {{"post", NULL, PURE_KIND_MESSAGE, false, both, 3}}; + groups.in = both_inputs; + CHECK(load(groups, &module_b) == PURE_INVALID && strstr(error, "field and group")); +} + +/* ---- Connections between apps (docs/research/078-app-connections.md): a + * send is part of its event and is seen once saved; a receiver applies each + * numbered message once, counts a refused one as heard, and forgets what a + * failed save undid. */ +enum { SEND = PURE_OP_SEND }; +typedef struct { uint32_t output; uint64_t seq; unsigned char payload[64]; size_t size; } Sent; +typedef struct { Sent items[16]; size_t count; } Outbox; + +static bool collect_sent(void *context, uint32_t output, uint64_t seq, PureSpan payload) { + Outbox *o = context; + CHECK(o->count < 16 && payload.size <= 64); + Sent *s = &o->items[o->count++]; + *s = (Sent){output, seq, {0}, payload.size}; + memcpy(s->payload, payload.data, payload.size); + return true; +} + +static Outbox outbox_of(const PureApp *app) { + Outbox o = {.count = 0}; + pure_app_outbox(app, collect_sent, &o); + return o; +} + +/* buy/fail {item: text, count: u32}: the count at 0, then the item. */ +static PureResult buy(PureApp *app, uint32_t input, const char *item, uint32_t count) { + unsigned char payload[64]; + size_t length = strlen(item); + memcpy(payload, &count, 4); + memcpy(payload + 4, item, length); + PureResult result = pure_app_message(app, input, NULL, -1, (PureSpan){payload, 4 + length}); + pure_app_finish(app); + return result; +} + +/* order {item: text, count: u32, total: number}: count, total, then item. */ +static PureResult deliver(PureApp *app, const char *source, uint64_t seq, const char *item, uint32_t count) { + unsigned char payload[64]; + size_t length = strlen(item); + double total = 0; + memcpy(payload, &count, 4); + memcpy(payload + 4, &total, 8); + memcpy(payload + 12, item, length); + PureResult result = pure_app_deliver(app, 0, source, seq, (PureSpan){payload, 12 + length}, NULL); + pure_app_finish(app); + return result; +} + +static void links(void) { + static const S sender_nums[] = {{"n", 0}}; + static const F order_fields[] = {{"item", PURE_FIELD_TEXT}, {"count", PURE_FIELD_U32}}; + static const F ordered_fields[] = {{"item", PURE_FIELD_TEXT}, {"count", PURE_FIELD_U32}, {"total", PURE_FIELD_NUMBER}}; + static const I sender_inputs[] = {{"buy", NULL, PURE_KIND_MESSAGE, false, order_fields, 2}, + {"fail", NULL, PURE_KIND_MESSAGE, false, order_fields, 2}, + {"ordered", NULL, PURE_KIND_OUTPUT, false, ordered_fields, 3}}; + static const O sender_code[] = {{L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, {FIELD, 0, 0}, {FIELD, 1, 0}, + {L, 0, 0}, {SEND, 2, 0}, /* buy */ + {FIELD, 0, 0}, {FIELD, 1, 0}, {C, 0, 0}, {SEND, 2, 0}, {C, 0, 0}, {REQ, 0, 0}}; /* fail */ + static const H sender_handlers[] = {{1, 0, 8}, {3, 8, 6}}; + Spec sender = {sender_nums, 1, sender_inputs, 3, NULL, 0, sender_handlers, 2, sender_code, LEN(sender_code), + NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; + static const S receiver_nums[] = {{"stock", 0}}; + static const T receiver_texts[] = {{"last", 1, 0}}; + static const char *const receiver_constants[] = {""}; + static const I receiver_inputs[] = {{"order", NULL, PURE_KIND_MESSAGE, false, ordered_fields, 3}}; + static const O receiver_code[] = {{FIELD, 1, 0}, {L, 0, 0}, {ADD, 0, 0}, {ST, 0, 0}, {FIELD, 0, 0}, {TSET, 1, 0}, + {FIELD, 1, 0}, {C, 0, 100}, {LT, 0, 0}, {REQ, 0, 0}}; /* refuses 100 or more */ + static const H receiver_handlers[] = {{1, 0, 10}}; + Spec receiver = {receiver_nums, 1, receiver_inputs, 1, NULL, 0, receiver_handlers, 1, receiver_code, + LEN(receiver_code), NULL, 0, receiver_texts, 1, receiver_constants, 1, NULL, 0, NULL, 0, NULL, 0}; + if (load(sender, &module_a) != PURE_OK || load(receiver, &module_b) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } + CHECK(module_a.input[2].prefix == 12); + + /* In memory: a send is its event's; the receiver applies it once per source. */ + PureApp a, b; + CHECK(pure_app_start(&a, &module_a, (PureData){0}) == PURE_OK && pure_app_start(&b, &module_b, (PureData){0}) == PURE_OK); + CHECK(buy(&a, 0, "pen", 2) == PURE_OK && buy(&a, 1, "ink", 3) == PURE_LIMIT); + Outbox o = outbox_of(&a); + CHECK(o.count == 1 && o.items[0].output == 2 && o.items[0].seq == 1 && o.items[0].size == 15 && a.values[0] == 1); + PureSpan sent = {o.items[0].payload, o.items[0].size}; + CHECK(pure_app_deliver(&b, 0, "orders.ordered", 1, sent, NULL) == PURE_OK && !b.duplicate && b.values[0] == 2 && + !strcmp(state_text(&b, 1), "pen")); + pure_app_finish(&b); + CHECK(pure_app_deliver(&b, 0, "orders.ordered", 1, sent, NULL) == PURE_OK && b.duplicate && b.values[0] == 2); + CHECK(deliver(&b, "other", 1, "cup", 5) == PURE_OK && !b.duplicate && b.values[0] == 7); + CHECK(deliver(&b, "orders.ordered", 3, "hat", 1) == PURE_OK && b.values[0] == 8); /* numbers may skip */ + CHECK(deliver(&b, "orders.ordered", 2, "old", 1) == PURE_OK && b.duplicate && b.values[0] == 8); + /* Refused (count 100): delivered all the same, never applied again. */ + CHECK(deliver(&b, "other", 2, "big", 100) == PURE_LIMIT && b.values[0] == 7 + 1); + CHECK(deliver(&b, "other", 2, "big", 1) == PURE_OK && b.duplicate && b.values[0] == 8); + /* Bad deliveries change nothing. */ + CHECK(deliver(&b, "", 4, "x", 1) == PURE_INVALID && deliver(&b, "a b", 4, "x", 1) == PURE_INVALID); + CHECK(deliver(&b, "other", 0, "x", 1) == PURE_INVALID && pure_app_deliver(&a, 2, "s", 1, sent, NULL) == PURE_INVALID); + CHECK(pure_app_message(&a, 2, NULL, -1, sent) == PURE_INVALID); + /* Trimmed, a send is gone at once for the host and from the store at the next save. */ + pure_app_trim(&a, 2, 1); + CHECK(outbox_of(&a).count == 0 && pure_app_save(&a) == PURE_OK && a.sent_count == 0); + CHECK(buy(&a, 0, "cap", 1) == PURE_OK && outbox_of(&a).items[0].seq == 2); + pure_app_stop(&a); + pure_app_stop(&b); + + /* Stored: sends and heard numbers survive restarts; a deferred burst's sends + * appear with its save, and a failed save takes them and their numbers back. */ + char path_a[64], path_b[64]; + snprintf(path_a, sizeof(path_a), "%s", new_store()); + snprintf(path_b, sizeof(path_b), "%s", new_store()); + CHECK(pure_app_start(&a, &module_a, pure_tree_data(path_a)) == PURE_OK); + CHECK(pure_app_start(&b, &module_b, pure_tree_data(path_b)) == PURE_OK); + CHECK(buy(&a, 0, "pen", 2) == PURE_OK && buy(&a, 0, "ink", 3) == PURE_OK); + pure_app_stop(&a); + CHECK(pure_app_start(&a, &module_a, pure_tree_data(path_a)) == PURE_OK); + o = outbox_of(&a); + CHECK(o.count == 2 && o.items[0].seq == 1 && o.items[1].seq == 2 && a.values[0] == 2); + CHECK(pure_app_defer(&a, true)); + for (int i = 0; i < 3; ++i) CHECK(buy(&a, 0, "box", 1) == PURE_OK); + CHECK(outbox_of(&a).count == 2 && pure_app_save(&a) == PURE_OK); + o = outbox_of(&a); + CHECK(o.count == 5 && o.items[4].seq == 5); + CHECK(buy(&a, 0, "lost", 1) == PURE_OK); + pure_app_trim(&a, 2, 3); + fail_stage = PURE_AFTER_HEADER; + CHECK(pure_app_save(&a) == PURE_STORE_ERROR); + fail_stage = -1; + o = outbox_of(&a); + CHECK(o.count == 5 && o.items[0].seq == 1 && a.values[0] == 5); /* the trim and the send undone */ + CHECK(buy(&a, 0, "next", 1) == PURE_OK && pure_app_save(&a) == PURE_OK); + o = outbox_of(&a); + CHECK(o.count == 6 && o.items[5].seq == 6); /* numbered on from 5 */ + pure_app_trim(&a, 2, 4); + CHECK(pure_app_save(&a) == PURE_OK); + pure_app_stop(&a); + CHECK(pure_app_start(&a, &module_a, pure_tree_data(path_a)) == PURE_OK); + o = outbox_of(&a); + CHECK(o.count == 2 && o.items[0].seq == 5 && o.items[1].seq == 6); + /* The receiver: heard numbers stay heard after a restart; a refused one too. */ + CHECK(deliver(&b, "orders.ordered", 1, "pen", 2) == PURE_OK && deliver(&b, "x", 1, "big", 100) == PURE_LIMIT); + pure_app_stop(&b); + CHECK(pure_app_start(&b, &module_b, pure_tree_data(path_b)) == PURE_OK && b.values[0] == 2); + CHECK(deliver(&b, "orders.ordered", 1, "pen", 2) == PURE_OK && b.duplicate && b.values[0] == 2); + CHECK(deliver(&b, "x", 1, "big", 1) == PURE_OK && b.duplicate); + /* A save that fails forgets what it heard. */ + CHECK(pure_app_defer(&b, true) && deliver(&b, "orders.ordered", 2, "ink", 3) == PURE_OK && b.values[0] == 5); + fail_stage = PURE_AFTER_HEADER; + CHECK(pure_app_save(&b) == PURE_STORE_ERROR && b.values[0] == 2); + fail_stage = -1; + CHECK(deliver(&b, "orders.ordered", 2, "ink", 3) == PURE_OK && !b.duplicate && b.values[0] == 5); + CHECK(pure_app_save(&b) == PURE_OK); + pure_app_stop(&b); + + /* Replacing: a module without the output is refused while sends are left + * on it; once they are trimmed it replaces, and the output's number is + * kept for a module that has it again. */ + static const I plain_inputs[] = {{"buy", NULL, PURE_KIND_MESSAGE, false, order_fields, 2}}; + static const O plain_code[] = {{L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}}; + static const H plain_handlers[] = {{1, 0, 4}}; + Spec plain = {sender_nums, 1, plain_inputs, 1, NULL, 0, plain_handlers, 1, plain_code, LEN(plain_code), + NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; + CHECK(load(plain, &module_b) == PURE_OK); + CHECK(pure_app_replace(&a, &module_b) == PURE_CONFLICT && a.module == &module_a); + pure_app_trim(&a, 2, 6); + CHECK(pure_app_save(&a) == PURE_OK && pure_app_replace(&a, &module_b) == PURE_OK); + CHECK(buy(&a, 0, "plain", 1) == PURE_OK && pure_app_save(&a) == PURE_OK); + pure_app_stop(&a); + CHECK(pure_app_start(&a, &module_b, pure_tree_data(path_a)) == PURE_OK); /* the number kept in the store */ + CHECK(pure_app_replace(&a, &module_a) == PURE_OK && buy(&a, 0, "back", 1) == PURE_OK && pure_app_save(&a) == PURE_OK); + o = outbox_of(&a); + CHECK(o.count == 1 && o.items[0].seq == 7); + /* A stored send whose output the module lacks refuses the start. */ + pure_app_stop(&a); + CHECK(pure_app_start(&a, &module_b, pure_tree_data(path_a)) == PURE_CONFLICT); + remove_store(path_a); + remove_store(path_b); + + /* A process that dies at each commit stage leaves an event and its send + * together or neither, and a delivery and its effect together or neither. */ + snprintf(path_a, sizeof(path_a), "%s", new_store()); + snprintf(path_b, sizeof(path_b), "%s", new_store()); + CHECK(load(receiver, &module_b) == PURE_OK); + uint64_t heard = 0; + size_t kept[2] = {0, 0}; /* sends and deliveries a death left in place */ + for (int round = 0; round < 2; ++round) + for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + PureApp doomed, taker; + if (pure_app_start(&doomed, &module_a, pure_tree_data(path_a)) != PURE_OK || + pure_app_start(&taker, &module_b, pure_tree_data(path_b)) != PURE_OK) _exit(1); + crash = 1; + fail_stage = stage; + if (round) buy(&doomed, 0, "crash", 1); + else deliver(&taker, "orders.ordered", heard + 1, "crash", 1); + _exit(0); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); + CHECK(pure_app_start(&a, &module_a, pure_tree_data(path_a)) == PURE_OK); + CHECK(pure_app_start(&b, &module_b, pure_tree_data(path_b)) == PURE_OK); + o = outbox_of(&a); + CHECK(o.count == (size_t)a.values[0] && (!o.count || o.items[o.count - 1].seq == o.count)); + heard = b.heard_count ? b.heard[0].seq : 0; + CHECK(b.values[0] == (double)heard); + kept[0] = (size_t)heard; + kept[1] = o.count; + pure_app_stop(&a); + pure_app_stop(&b); + } + CHECK(kept[0] == 1 && kept[1] == 1); /* only a death after the head was written keeps it */ + remove_store(path_a); + remove_store(path_b); + + /* What an output and SEND may not be. */ + const struct { const O *code; size_t count; const char *reason; } refused[] = { + {(const O[]){{FIELD, 1, 0}, {FIELD, 0, 0}, {L, 0, 0}, {SEND, 2, 0}}, 4, "operand type"}, + {(const O[]){{FIELD, 0, 0}, {FIELD, 1, 0}, {L, 0, 0}, {SEND, 1, 0}}, 4, "no such state"}, + {(const O[]){{FIELD, 0, 0}, {FIELD, 1, 0}, {SEND, 2, 0}}, 3, "stack underflow"}, + }; + for (size_t i = 0; i < LEN(refused); ++i) { + Spec bad = sender; + H handler = {1, 0, (uint32_t)refused[i].count}; + bad.o = refused[i].code; bad.no = refused[i].count; bad.h = &handler; bad.nh = 1; + if (load(bad, &module_b) != PURE_INVALID || !strstr(error, refused[i].reason)) { + fprintf(stderr, "links case %zu: expected '%s', got '%s'\n", i, refused[i].reason, error); + abort(); + } + } + H on_output[] = {{1, 0, 8}, {5, 8, 6}}; + Spec bad = sender; + bad.h = on_output; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "handler event")); +} + +/* A timer's tick sees the time the host set; floor and a floored modulo + * take it apart. now is for handlers only. */ +enum { NOW = PURE_OP_NOW, FLOOR = PURE_OP_FLOOR, MOD = PURE_OP_MOD }; +static void timers(void) { + static const S nums[] = {{"clock", 0}, {"seconds", 0}, {"ticks", 0}, {"rest", 0}}; + static const I inputs[] = {TIMER("tick", 1000), KEY("left", "KeyL"), KEY("zero", "KeyZ")}; + static const O code[] = { + {NOW, 0, 0}, {ST, 0, 0}, {NOW, 0, 0}, {C, 0, 1000}, {DIV, 0, 0}, {FLOOR, 0, 0}, {C, 0, 60}, {MOD, 0, 0}, + {ST, 1, 0}, {L, 2, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 2, 0}, /* 0 tick */ + {C, 0, -7}, {C, 0, 3}, {MOD, 0, 0}, {C, 0, 7}, {C, 0, -3}, {MOD, 0, 0}, {C, 0, 10}, {MUL, 0, 0}, + {ADD, 0, 0}, {ST, 3, 0}, /* 13 left: 2 + -20 */ + {C, 0, 1}, {C, 0, 0}, {MOD, 0, 0}, {ST, 3, 0}, /* 23 zero */ + }; + static const H handlers[] = {{1, 0, 13}, {3, 13, 10}, {5, 23, 4}}; + static const V one_view[] = {{"v", 0, 1}}; + Spec spec = SPEC(nums, inputs, one_view, handlers, code); + spec.nv = 0; + if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } + CHECK(module_a.input[0].kind == PURE_KIND_TIMER && module_a.input[0].period == 1000); + PureApp app; + CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); + app.now = 1700000123456.0; + CHECK(pure_app_tick(&app, 0) == PURE_OK && app.values[0] == 1700000123456.0 && + app.values[1] == (double)(1700000123 % 60) && app.values[2] == 1); + CHECK(pure_app_tick(&app, 1) == PURE_INVALID && pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_INVALID); + CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[3] == 2 - 20); + /* A zero takes the divisor's sign too. */ + static const O zeros[] = {{C, 0, -6}, {C, 0, 3}, {MOD, 0, 0}, {ST, 0, 0}, {C, 0, 6}, {C, 0, -3}, {MOD, 0, 0}, {ST, 1, 0}}; + static const H zero_handler[] = {{3, 0, 8}}; + Spec signs = spec; + signs.o = zeros; signs.no = 8; signs.h = zero_handler; signs.nh = 1; + CHECK(load(signs, &module_b) == PURE_OK); + PureApp zero; + CHECK(pure_app_start(&zero, &module_b, (PureData){0}) == PURE_OK && pure_app_input(&zero, 1, PURE_EDGE_DOWN) == PURE_OK); + CHECK(zero.values[0] == 0 && !signbit(zero.values[0]) && zero.values[1] == 0 && signbit(zero.values[1])); + pure_app_stop(&zero); + CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_INVALID && app.fault == PURE_FAULT_DIVIDE); + pure_app_stop(&app); + /* A timer's period is 1 ms to a day, and it has nothing else; now is for handlers. */ + static const I no_period[] = {TIMER("tick", 0)}, too_long[] = {TIMER("tick", PURE_TIMER_LONGEST + 1)}; + static const O now_view[] = {{NOW, 0, 0}}; + Spec bad = spec; + bad.in = no_period; bad.ni = 1; bad.nh = 1; bad.o = code; bad.no = 13; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "timer input")); + bad.in = too_long; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "timer input")); + bad = SPEC(nums, inputs, one_view, handlers, now_view); + bad.nh = 0; + CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view with an effect")); +} + +int main(void) { + behaviour(); + faults(); + rejections(); + mutations(); + messages(); + message_fields(); + timers(); + message_mutations(); + texts(); + marks(); + text_mutations(); + lists(); + folds(); + filters(); + keyed(); + keyed_mutations(); + snapshots(); + links(); + fold_mutations(); + pointing(); + list_mutations(); + tables(); + table_mutations(); + stored_states(); + deferred(); + printf("{\"sizeof_module\":%zu,\"sizeof_app\":%zu}\n", sizeof(PureModule), sizeof(PureApp)); + puts("{\"module_tests\":\"verified-load-atomic-events-faults-signed-zero-if-else-replace-by-name-rejections-mutations-messages-documents-typed-verification-text-cursors-utf8-views-rollback-replace-lists-indices-undo-migration-joins-tables-columns-pick-slide-stored-equivalence-crashes-frames-messages-migration-deferred-bursts-pointing-marks-folds-filters-keyed-lists-text-order-message-fields-timers-now-snapshots-links\"}"); + return 0; +} diff --git a/docs/design/008-module-format.md b/docs/design/008-module-format.md index 3fd4d52..84aa97b 100644 --- a/docs/design/008-module-format.md +++ b/docs/design/008-module-format.md @@ -1,7 +1,7 @@ # Pure 모듈 형식 v5 -2026-09-25. 구현 `prototypes/connections/module.{h,c}`, `module-text.c`, `module-list.c`, 상태 저장 `module-save.c`, -트리 저장소 연결 `module-store.c`. +2026-09-25. 구현 `core/module.{h,c}`, `module-text.c`, `module-list.c`, 상태 저장 `module-save.c`, +모듈 파일 읽기 `module-store.c`, 트리 저장소 `store/store-tree.c`, HTTP 호스트 `serve/`. Pure가 실행하는 앱 정의의 형식이다. JVM이 class 파일 형식을 소유하듯 Pure가 이 형식과 검증 규칙을 소유한다. 생산자는 [aup](/srv/git/aup.git) 같은 툴체인이다. Pure는 소스·문법을 모르며, diff --git a/docs/design/009-layout.md b/docs/design/009-layout.md new file mode 100644 index 0000000..8cb9a78 --- /dev/null +++ b/docs/design/009-layout.md @@ -0,0 +1,60 @@ +# 009 — 배치: 코어와 고르는 부품 + +2026-09-27. 저장소 경계와 범위가 일관되지 않다는 지적에서 정한 것. + +## 문제 + +- README의 원칙은 "항상 있는 코어는 실행, 소유권, 자원 한도, 완료 규칙만 다룬다. 프로토콜, 저장소 구현, UI, 코덱은 앱에 + 맞게 고른다"였다. 그런데 소스는 원칙과 다르게 놓여 있었다. +- 지금 세대의 코어(모듈 검증·실행 `module*.c`), 트리 저장소(`store-tree.c`), HTTP 호스트(`serve.c`, `deliver.c`, + `message-json.c`)가 모두 `prototypes/connections/` 한 디렉터리에 이전 세대(000–077의 문서 런타임: `program.c`, + `store.c`, `server.c`, 편집기들)와 섞여 있었다. 이름만 보면 저장소는 형식 매직(`PURETRE2`)으로만 드러났고, 코어가 + 어디서 끝나는지 알 수 없었다. +- 소비자는 소스 파일 목록으로 연결했다. Vega는 Pure의 `.c` 여섯 개를 경로로 나열해 컴파일했다. Pure에 파일이 하나 + 늘면 Vega가 깨진다. +- 반면 사람을 위한 인터페이스(Vega)는 따로 된 저장소였다. 저장소와 네트워크는 "호스트 안"처럼 보이고 UI만 부품처럼 + 보였다. + +## 규칙 + +1. **역할로 나누고 역할로 부른다.** 코어, 저장소, HTTP 호스트, 명령행 호스트. 형식 이름(`PURETRE3`, `PUREREC1`)은 + 형식의 이름일 뿐 부품의 이름이 아니다. +2. **부품은 라이브러리와 헤더로 쓴다.** 소비자는 파일 목록이 아니라 `libpure.a`와 `core/`의 헤더로 연결한다. +3. **저장소(git) 경계는 결합도로 정한다.** 함께 바뀌는 것은 한 저장소에 두고 디렉터리로 나누며, 따로 바뀌는 것은 저장소를 + 나눈다. 부품이 된 디렉터리는 나중에 `git subtree split`으로 그대로 떼어 낼 수 있다. + +## 배치 + +| 디렉터리 | 역할 | 만드는 것 | +|---|---|---| +| `core/` | 실행·소유권·한도·완료(`runtime.c`, `flow.c`), 앱 모델(`module*.c`: 실행 전에 검증한 모듈, 트랜잭션인 이벤트, draft인 상태), 코어가 저장소에 요구하는 것(`state.h`, `store.h`) | `build/libpure.a`(저장소와 함께) | +| `store/` | 트리 저장소. 한 파일, 커밋은 한 연속 쓰기와 동기화 한 번 | `libpure.a` 안 | +| `serve/` | `pure-serve`: HTTP 호스트, 나가는 전달, 제자리 JSON | `build/pure-serve` | +| `cli/` | 명령행 호스트 | `build/pure-run`, `build/pure-edit-module` | +| `prototypes/` | 이전 문서 런타임. `core/`와 `store/`를 가져다 빌드한다 | `make -C prototypes/connections` | + +저장소 계약(`state.h`, `store.h`)은 코어 쪽에 둔다. 코어가 필요한 것을 정하고 저장소 구현이 그것을 채운다. 트리 +저장소는 그 계약의 한 구현이다(이전 세대의 `store.c`, `store-paged.c`도 `store.h`의 구현이다). + +## 왜 저장소를 따로 된 git 저장소로 떼지 않았나 + +트리 저장소를 바꾼 최근 커밋 12개 중 9개가 계약(`state.h`, `store.h`)을 바꿨고, 그중 5개는 같은 커밋에서 코어나 호스트도 +바꿨다. 스냅숏(`571824a`: 저장소 동결 + 모듈 리스트 공유 + 모듈 스냅숏), 대여(`1a3586d`: 저장소 익스텐트 + serve), 그룹 +커밋(`d4b01a7`), 샤딩(`694b60f`), 매핑을 나누는 핸들(`89d1777`: + 모듈 파일 읽기). 계약이 아직 코어와 함께 움직이고 +소비자가 Pure 하나뿐이다. 따로 된 저장소면 이런 +변경마다 두 저장소의 버전을 맞춰야 한다. 두 번째 소비자(예: 메신저의 메시지 저장)가 생기거나 계약이 멈추면 `store/`를 +그대로 떼어 낸다. + +## 왜 Vega는 따로이고 이름이 pure-ui가 아닌가 + +Vega는 두 가지다. aup 라이브러리(`use vega`: 목록·표·포커스·톤 같은 인터페이스 어휘)와 그 어휘를 사람에게 보이는 +표면들(TUI, CLI, NUI, 웹, 심볼과 GPU)이다. 어휘는 aup의 언어 기능과 함께 바뀌고, 표면은 Pure의 모듈 API(`libpure.a`) +위에 있다. Vega의 최근 커밋은 모두 Vega 안에서 끝났다. `pure-serve`가 Pure 안에 있는 것은 프로그램을 위한 표면(보기가 +GET, 입력이 POST)에는 어휘가 필요 없기 때문이다. 모듈 형식만으로 정해진다. 사람을 위한 표면은 어휘가 필요하다. JDK가 +`java.net`과 HTTP 서버를 담고 JavaFX를 따로 두는 것과 같은 선이다. `pure-ui`라는 이름은 UI가 런타임의 일부라는 뜻이 +되어, 런타임이 화면을 모른다는 분리와 어긋난다. + +## 확인 + +최상위 `make check`(저장소와 앱 모델의 검사), `prototypes/connections`의 `make check`·`check-tree`·`check-core`, aup +`make check`, Vega `make check`가 새 배치에서 통과한다. Vega는 이제 `libpure.a`를 링크한다. diff --git a/docs/research/018-native-connections.md b/docs/research/018-native-connections.md index f2c26bc..6747dd8 100644 --- a/docs/research/018-native-connections.md +++ b/docs/research/018-native-connections.md @@ -66,7 +66,7 @@ DB 호출은 아직 동기식이므로 디스크 작업 중 다른 연결 처리 동적 링크한 SQLite 자체를 sanitizer로 다시 빌드한 것은 아니다. [원자료·재현](../../prototypes/connections/results/2026-09-23-native-connections/README.md), -[전송 수명](../../prototypes/connections/flow.c), +[전송 수명](../../core/flow.c), [DB 연결](../../prototypes/connections/store.c), [호스트](../../prototypes/connections/server.c), [기존 CLI 연결](../../prototypes/workbench/remote.mjs). diff --git a/docs/research/019-byte-storage.md b/docs/research/019-byte-storage.md index faefecf..b58adcd 100644 --- a/docs/research/019-byte-storage.md +++ b/docs/research/019-byte-storage.md @@ -72,7 +72,7 @@ Linux AArch64에서 일반 빌드와 ASan·UBSan 빌드가 모두 통과했다. [원자료와 재현](../../prototypes/connections/results/2026-09-23-byte-storage/README.md), [직접 저장 구현](../../prototypes/connections/store.c), -[구간·수명 API](../../prototypes/connections/store.h). +[구간·수명 API](../../core/store.h). 018의 원자료는 당시 SQLite 구현의 기록으로 별도 보존한다. ## 다음 결정 diff --git a/docs/research/030-authenticated-document-peer.md b/docs/research/030-authenticated-document-peer.md index 1ff569d..6e1fa23 100644 --- a/docs/research/030-authenticated-document-peer.md +++ b/docs/research/030-authenticated-document-peer.md @@ -9,7 +9,7 @@ `pure-document-peer`는 TLS 1.3/TCP의 상호 인증, 인증서 SAN 이름 확인, `pure-doc/1` ALPN 뒤에 자체 이진 메시지를 사용한다. 이미 구현한 scalar 피어의 소켓·TLS 부분을 [`tls-link.c`](../../prototypes/common/tls-link.c)로 분리해 두 피어가 재사용한다. -코어의 [`runtime.c`](../../prototypes/connections/runtime.c), 저장소, `editor.pure`는 +코어의 [`runtime.c`](../../core/runtime.c), 저장소, `editor.pure`는 문서 CLI와 같다. 전송 코드는 선택 빌드의 호스트에만 있다. 서버가 보내는 40 B hello는 `PDOC`, 버전 1, 예약 3 B, **원본 .pure 소스** SHA-256이다. diff --git a/docs/research/064-release-callback-reentrancy.md b/docs/research/064-release-callback-reentrancy.md index 68ce819..f35f6ec 100644 --- a/docs/research/064-release-callback-reentrancy.md +++ b/docs/research/064-release-callback-reentrancy.md @@ -5,7 +5,7 @@ `running`이 꺼져 있었다. 콜백이 같은 영역의 티켓을 해제하거나 새 작업을 받아들이면, 바깥쪽 회수 코드가 작업 슬롯과 예산 장부를 계속 수정할 수 있었다. -[`runtime.c`](../../prototypes/connections/runtime.c)는 자원 해제 콜백 동안 해당 +[`runtime.c`](../../core/runtime.c)는 자원 해제 콜백 동안 해당 영역을 실행 중으로 표시하고 이전 상태를 복원한다. 새 상태 필드나 작업별 할당은 없다. [`test-runtime.c`](../../prototypes/connections/test-runtime.c)는 취소, 대기 작업 해제, 유효하지 않은 호스트 할당 반환에서 콜백이 같은 티켓을 diff --git a/docs/research/066-application-boundary.md b/docs/research/066-application-boundary.md index 44e6d86..bb96bed 100644 --- a/docs/research/066-application-boundary.md +++ b/docs/research/066-application-boundary.md @@ -5,7 +5,7 @@ ## 현재 코드가 증명하는 것과 못 하는 것 -`prototypes/connections/runtime.c`는 입력 접수, 바이트 예산, 작업 슬롯, 완료, +`core/runtime.c`는 입력 접수, 바이트 예산, 작업 슬롯, 완료, 대여와 다른 영역으로의 무복사 전달을 소유한다. 하지만 실행 단계에서 `PureProgram`의 저장 전용 `read`/`write`/`splice`를 직접 해석하고 `PureHost.read`/`write`/`splice`를 호출한다. 따라서 현재 5,980 B 코어는 diff --git a/docs/research/073-single-file-persistent-tree.md b/docs/research/073-single-file-persistent-tree.md index 13884e3..4386e93 100644 --- a/docs/research/073-single-file-persistent-tree.md +++ b/docs/research/073-single-file-persistent-tree.md @@ -74,7 +74,7 @@ Ctrl+S부터 저장 완료 프레임까지 잰다. 1 MiB 문서, 저장소별 3 ## 검증 -[`test-tree-store.c`](../../prototypes/connections/test-tree-store.c)는 ASan·UBSan에서도 통과했다. +[`test-tree-store.c`](../../store/test-tree-store.c)는 ASan·UBSan에서도 통과했다. - 4개 키에 6,000회 무작위 편집을 평면 배열 오라클과 매번 비교했다(최대 333,510 B, 높이 2). 매 편집 뒤 저장된 모든 비루트 노드가 기준 이상으로 채워졌는지 확인했다. diff --git a/docs/research/074-runtime-owned-drafts.md b/docs/research/074-runtime-owned-drafts.md index 71ac8c9..42fd417 100644 --- a/docs/research/074-runtime-owned-drafts.md +++ b/docs/research/074-runtime-owned-drafts.md @@ -1,7 +1,7 @@ # 074 — 작업 중인 값을 런타임이 소유한다 2026-09-25. [073](073-single-file-persistent-tree.md)의 선택형 트리 저장소 위에 런타임 API -[`state.h`](../../prototypes/connections/state.h)를 추가했다. 기본 빌드와 기본 저장 형식은 바뀌지 않았다. +[`state.h`](../../core/state.h)를 추가했다. 기본 빌드와 기본 저장 형식은 바뀌지 않았다. Pure는 JVM과 같은 위치의 런타임이다. 앱 기능은 별도 저장소에서 만들고, 이 저장소의 CLI·TUI· HTTP 호스트는 런타임을 검사하는 호스트다. 따라서 이번 단계는 편집기 기능이 아니라 모든 앱이 쓰는 @@ -61,7 +61,7 @@ draft가 가진 옛 매핑에서 필요한 노드를 새 파일로 복사한다( ## 검증 -[`test-tree-store.c`](../../prototypes/connections/test-tree-store.c) 추가 검사는 ASan·UBSan을 포함해 통과했다. +[`test-tree-store.c`](../../store/test-tree-store.c) 추가 검사는 ASan·UBSan을 포함해 통과했다. - 20,000회 무작위 편집을 draft에서 평면 오라클과 비교하고 1,000회마다 커밋했다(최종 419,324 B, 커밋 20회). 커밋 뒤에는 저장본 일치, revision 증가, 빈 arena를 확인했다. diff --git a/docs/research/075-http-host-against-axum.md b/docs/research/075-http-host-against-axum.md index 17cfe39..77be305 100644 --- a/docs/research/075-http-host-against-axum.md +++ b/docs/research/075-http-host-against-axum.md @@ -55,7 +55,7 @@ PUT은 연결마다 자기 키 앞에 16 B를 넣고 받은 ETag로 다음 `If-M ## 1단계 결과: `pure-serve` (소유 스레드 하나) -새 호스트 [`serve.c`](../../prototypes/connections/serve.c): 지속 연결·파이프라이닝, edge-triggered epoll, 연결 +새 호스트 [`serve.c`](../../serve/serve.c): 지속 연결·파이프라이닝, edge-triggered epoll, 연결 최대 1,024(받을 때 할당), `writev` 응답, 같은 런타임 코어와 묶음 커밋. 같은 조건에서 다시 쟀다. `strace -c`로 보니 GET 하나마다 저장소 draft를 여닫느라 시스템 호출이 9번(파일·디렉터리 열기, `fstat` 두 번, diff --git a/docs/research/076-one-sync-commit.md b/docs/research/076-one-sync-commit.md index 9f010e7..4b9afa6 100644 --- a/docs/research/076-one-sync-commit.md +++ b/docs/research/076-one-sync-commit.md @@ -94,7 +94,7 @@ p99는 장치 쪽 잡음이 지배한다. 꼬리를 늘리는 단위를 64 KiB, ## 검증 -[`test-tree-store.c`](../../prototypes/connections/test-tree-store.c)는 ASan·UBSan에서도 통과했다. 새로 넣은 검사: +[`test-tree-store.c`](../../store/test-tree-store.c)는 ASan·UBSan에서도 통과했다. 새로 넣은 검사: - **잃어버린 커밋**: 최신 커밋의 바이트 하나를 바꾸고 프로세스의 기억을 지우면(재부팅 뒤의 새 프로세스와 같다) 읽기, 핸들, writer가 모두 이전 커밋을 쓴다. 다음 커밋은 잃어버린 커밋이 있던 자리부터 쓴다. diff --git a/prototypes/connections/Makefile b/prototypes/connections/Makefile index 2dd85c7..9c9f485 100644 --- a/prototypes/connections/Makefile +++ b/prototypes/connections/Makefile @@ -2,19 +2,23 @@ CC ?= cc CFLAGS ?= -O2 -g CFLAGS += -std=c17 -Wall -Wextra -Wpedantic -Werror BUILD ?= build - -.PHONY: all core async-core editor fixed-editor fixed-server fixed-tui source-tui compiler mirror window server async-server document-peer document-server check check-policy check-policy-peer check-portable check-portable-tui check-paged-portable check-tree check-module check-portable-peer check-native-checkpoint check-mirror check-core check-async check-async-server check-async-service-scale check-profile check-fixed-profile check-fixed-server check-service-scale check-tui-profile check-source-tui-profile check-tui check-source-tui check-release check-micro-memory check-hybrid check-flat-export check-copy-range check-peer-profile check-document-server-profile check-window-profile check-document-peer check-document-server FORCE -CORE = runtime.c +# The core (runtime, flow, the storage contracts) and the tree store now live +# in ../../core and ../../store; this earlier document runtime builds on them. +PURE_ROOT = ../.. +CFLAGS += -I$(PURE_ROOT)/core +FLOW = $(PURE_ROOT)/core/flow.c + +.PHONY: all core async-core editor fixed-editor fixed-server fixed-tui source-tui compiler mirror window server async-server document-peer document-server check check-policy check-policy-peer check-portable check-portable-tui check-paged-portable check-tree check-portable-peer check-native-checkpoint check-mirror check-core check-async check-async-server check-async-service-scale check-profile check-fixed-profile check-fixed-server check-service-scale check-tui-profile check-source-tui-profile check-tui check-source-tui check-release check-micro-memory check-hybrid check-flat-export check-copy-range check-peer-profile check-document-server-profile check-window-profile check-document-peer check-document-server FORCE +CORE = $(PURE_ROOT)/core/runtime.c STORE ?= store.c FIXED_PROGRAM ?= examples/editor.pure -HOST = $(STORE) store-port.c evaluate.c +STORE_FILE = $(if $(filter store-tree.c,$(STORE)),$(PURE_ROOT)/store/store-tree.c,$(STORE)) +HOST = $(STORE_FILE) store-port.c evaluate.c # Document browsers list keys through the store. The tree store keeps its own # catalog; per-object file stores share a directory listing. KEYS = $(if $(filter store-tree.c,$(STORE)),,store-keys.c) -HEADERS = runtime.h program.h store.h store-port.h state.h flow.h result.h -# Defined before any rule names them: prerequisites expand when a rule is read. -MODULE = module.c module-list.c module-save.c module-text.c store-tree.c -MODULE_HEADERS = module.h module-list.h module-save.h module-text.h $(HEADERS) +CORE_HEADERS = $(addprefix $(PURE_ROOT)/core/,runtime.h store.h state.h flow.h result.h) +HEADERS = $(CORE_HEADERS) program.h store-port.h all: core fixed-editor core: $(BUILD)/libpure-execution.a async-core: $(BUILD)/libpure-execution-async.a @@ -34,21 +38,17 @@ document-server: $(BUILD)/pure-document-server $(BUILD): mkdir -p $@ -$(BUILD)/pure-connections: server.c $(HOST) flow.c program.c $(CORE) $(HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_RUNTIME_METRICS server.c $(HOST) flow.c program.c $(CORE) $(LDFLAGS) $(LDLIBS) -o $@ - -$(BUILD)/pure-connections-async: server.c $(HOST) flow.c program.c $(CORE) $(HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_RUNTIME_METRICS -DPURE_ASYNC_EFFECTS -DPURE_HTTP_ASYNC server.c $(HOST) flow.c program.c $(CORE) $(LDFLAGS) -pthread $(LDLIBS) -o $@ +$(BUILD)/pure-connections: server.c $(HOST) $(FLOW) program.c $(CORE) $(HEADERS) | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_RUNTIME_METRICS server.c $(HOST) $(FLOW) program.c $(CORE) $(LDFLAGS) $(LDLIBS) -o $@ -$(BUILD)/pure-connections-async-gated: server.c $(HOST) flow.c program.c $(CORE) test-async-server-gate.c $(HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_RUNTIME_METRICS -DPURE_ASYNC_EFFECTS -DPURE_HTTP_ASYNC -DPURE_STORE_TEST server.c $(HOST) flow.c program.c $(CORE) test-async-server-gate.c $(LDFLAGS) -pthread $(LDLIBS) -o $@ +$(BUILD)/pure-connections-async: server.c $(HOST) $(FLOW) program.c $(CORE) $(HEADERS) | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_RUNTIME_METRICS -DPURE_ASYNC_EFFECTS -DPURE_HTTP_ASYNC server.c $(HOST) $(FLOW) program.c $(CORE) $(LDFLAGS) -pthread $(LDLIBS) -o $@ -# The HTTP host for aup modules: persistent connections, epoll, group commits. -$(BUILD)/pure-serve: serve.c deliver.c message-json.c flow.c module-store.c $(MODULE) $(CORE) deliver.h message-json.h module-store.h $(MODULE_HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_ASYNC_EFFECTS serve.c deliver.c message-json.c flow.c module-store.c $(MODULE) $(CORE) $(LDFLAGS) -lssl -lcrypto -lm $(LDLIBS) -o $@ +$(BUILD)/pure-connections-async-gated: server.c $(HOST) $(FLOW) program.c $(CORE) test-async-server-gate.c $(HEADERS) | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_RUNTIME_METRICS -DPURE_ASYNC_EFFECTS -DPURE_HTTP_ASYNC -DPURE_STORE_TEST server.c $(HOST) $(FLOW) program.c $(CORE) test-async-server-gate.c $(LDFLAGS) -pthread $(LDLIBS) -o $@ -$(BUILD)/pure-connections-fixed: server.c $(HOST) flow.c $(CORE) $(HEADERS) $(BUILD)/embedded-plan.h | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_RUNTIME_METRICS -DPURE_EMBEDDED_PROGRAM -I$(BUILD) server.c $(HOST) flow.c $(CORE) $(LDFLAGS) $(LDLIBS) -o $@ +$(BUILD)/pure-connections-fixed: server.c $(HOST) $(FLOW) $(CORE) $(HEADERS) $(BUILD)/embedded-plan.h | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_RUNTIME_METRICS -DPURE_EMBEDDED_PROGRAM -I$(BUILD) server.c $(HOST) $(FLOW) $(CORE) $(LDFLAGS) $(LDLIBS) -o $@ $(BUILD)/pure-edit: edit.c $(HOST) program.c $(CORE) $(HEADERS) | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) edit.c $(HOST) program.c $(CORE) $(LDFLAGS) $(LDLIBS) -o $@ @@ -87,13 +87,13 @@ $(BUILD)/pure-edit-window: edit-window.c $(HOST) program.c $(CORE) $(HEADERS) | pkg-config --atleast-version=4.10 gtk4 $(CC) $(CPPFLAGS) $(CFLAGS) $$(pkg-config --cflags gtk4) edit-window.c $(HOST) program.c $(CORE) $(LDFLAGS) $$(pkg-config --libs gtk4) $(LDLIBS) -o $@ -$(BUILD)/runtime.o: runtime.c $(HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -c runtime.c -o $@ +$(BUILD)/runtime.o: $(CORE) $(HEADERS) | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) -c $(CORE) -o $@ -$(BUILD)/runtime-async.o: runtime.c $(HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_ASYNC_EFFECTS -c runtime.c -o $@ +$(BUILD)/runtime-async.o: $(CORE) $(HEADERS) | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_ASYNC_EFFECTS -c $(CORE) -o $@ -$(BUILD)/evaluate.o: evaluate.c program.h flow.h | $(BUILD) +$(BUILD)/evaluate.o: evaluate.c program.h $(PURE_ROOT)/core/flow.h | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) -c evaluate.c -o $@ $(BUILD)/libpure-execution.a: $(BUILD)/runtime.o @@ -104,13 +104,13 @@ $(BUILD)/libpure-execution-async.a: $(BUILD)/runtime-async.o rm -f $@ $(AR) rcs $@ $^ -$(BUILD)/test-flow: test-flow.c store.c flow.c store.h flow.h result.h | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_STORE_TEST test-flow.c store.c flow.c $(LDFLAGS) $(LDLIBS) -o $@ +$(BUILD)/test-flow: test-flow.c store.c $(FLOW) $(PURE_ROOT)/core/store.h $(PURE_ROOT)/core/flow.h $(PURE_ROOT)/core/result.h | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_STORE_TEST test-flow.c store.c $(FLOW) $(LDFLAGS) $(LDLIBS) -o $@ -$(BUILD)/test-program: test-program.c program.c evaluate.c program.h store.h flow.h result.h | $(BUILD) +$(BUILD)/test-program: test-program.c program.c evaluate.c program.h $(PURE_ROOT)/core/store.h $(PURE_ROOT)/core/flow.h $(PURE_ROOT)/core/result.h | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) test-program.c program.c evaluate.c $(LDFLAGS) $(LDLIBS) -o $@ -$(BUILD)/test-fields: test-fields.c program.c evaluate.c program.h store.h flow.h result.h | $(BUILD) +$(BUILD)/test-fields: test-fields.c program.c evaluate.c program.h $(PURE_ROOT)/core/store.h $(PURE_ROOT)/core/flow.h $(PURE_ROOT)/core/result.h | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) test-fields.c program.c evaluate.c $(LDFLAGS) $(LDLIBS) -o $@ $(BUILD)/test-runtime: test-runtime.c program.c $(CORE) $(HOST) $(HEADERS) | $(BUILD) @@ -122,24 +122,8 @@ $(BUILD)/test-async: test-async.c program.c $(CORE) $(HOST) $(HEADERS) | $(BUILD $(BUILD)/test-splice: test-splice.c program.c $(CORE) $(HOST) $(HEADERS) | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_RUNTIME_METRICS -DPURE_STORE_TEST test-splice.c program.c $(CORE) $(HOST) $(LDFLAGS) $(LDLIBS) -o $@ -# Pure modules: loader/verifier/executor, its tests and a test host. -# Text states are runtime-owned drafts, so the executor links the tree engine. - -$(BUILD)/test-module: test-module.c module-store.c $(MODULE) module-store.h $(MODULE_HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_STORE_TEST test-module.c module-store.c $(MODULE) $(LDFLAGS) -pthread -lm $(LDLIBS) -o $@ - -$(BUILD)/pure-run: pure-run.c module-store.c $(MODULE) module-store.h $(MODULE_HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) pure-run.c module-store.c $(MODULE) $(LDFLAGS) -lm $(LDLIBS) -o $@ - -# The document CLI contract driven by a module on the tree store. -$(BUILD)/pure-edit-module: edit-module.c module-store.c $(MODULE) module-store.h $(MODULE_HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) edit-module.c module-store.c $(MODULE) $(LDFLAGS) -lm $(LDLIBS) -o $@ - -check-module: $(BUILD)/test-module $(BUILD)/pure-run $(BUILD)/pure-edit-module $(BUILD)/pure-serve - $(BUILD)/test-module - -$(BUILD)/test-tree-store: test-tree-store.c store-tree.c $(HEADERS) | $(BUILD) - $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_STORE_TEST test-tree-store.c store-tree.c $(LDFLAGS) $(LDLIBS) -o $@ +$(BUILD)/test-tree-store: $(PURE_ROOT)/store/test-tree-store.c $(PURE_ROOT)/store/store-tree.c $(HEADERS) | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_STORE_TEST $(PURE_ROOT)/store/test-tree-store.c $(PURE_ROOT)/store/store-tree.c $(LDFLAGS) $(LDLIBS) -o $@ $(BUILD)/test-paged-store: test-paged-store.c store-paged.c $(HEADERS) | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) -DPURE_STORE_TEST test-paged-store.c store-paged.c $(LDFLAGS) $(LDLIBS) -o $@ @@ -248,7 +232,7 @@ check: $(BUILD)/test-flow $(BUILD)/test-program $(BUILD)/test-fields $(BUILD)/te $(BUILD)/test-flow $(BUILD)/test-program $(BUILD)/test-fields - python3 check.py --server $(BUILD)/pure-connections --runtime-test $(BUILD)/test-runtime --splice-test $(BUILD)/test-splice --editor $(BUILD)/pure-edit --store-source $(STORE) $(if $(EVIDENCE),--output "$(EVIDENCE)") + python3 check.py --server $(BUILD)/pure-connections --runtime-test $(BUILD)/test-runtime --splice-test $(BUILD)/test-splice --editor $(BUILD)/pure-edit --store-source $(STORE_FILE) $(if $(EVIDENCE),--output "$(EVIDENCE)") check-policy: editor server source-tui $(MAKE) BUILD=$(BUILD)/policy FIXED_PROGRAM=examples/bounded-editor.pure fixed-editor fixed-server fixed-tui diff --git a/prototypes/connections/README.md b/prototypes/connections/README.md index 94d05f0..569fe41 100644 --- a/prototypes/connections/README.md +++ b/prototypes/connections/README.md @@ -1,5 +1,10 @@ # Pure: 작성한 연결을 준비해 실행하는 네이티브 경로 +2026-09-27부터 실행 코어(`runtime.c`, `flow.c`, 저장소 계약 `store.h`·`state.h`)와 트리 저장소는 저장소 최상위의 +[`core/`](../../core)와 [`store/`](../../store)에 있다. 여기의 이전 문서 런타임(000–077)은 그것들을 가져다 빌드한다 +(`make -C prototypes/connections check`, `check-tree`). 모듈 앱의 호스트(`pure-serve`, `pure-run`)는 최상위 `make`가 +만든다. [배치](../../docs/design/009-layout.md). + 기존 Workbench에서 만든 코드·배치를 실제 TCP 요청으로 저장하고 새 실행기에서 복원한다. 서버는 C17 네이티브 실행 파일이다. SQL·SQLite 의존성을 제거했고, 호출자의 바이트를 그대로 파일에 기록한다. 저장할 구간과 응답할 구간은 `.pure` 파일로 작성하며 서버 시작 때 diff --git a/prototypes/connections/check.py b/prototypes/connections/check.py index 7719537..8a51858 100644 --- a/prototypes/connections/check.py +++ b/prototypes/connections/check.py @@ -500,6 +500,12 @@ def rpc_bytes(path): return json.dumps(json.loads(path.read_text()), ensure_ascii=False, separators=(',', ':')).encode() +def source(name): + """A source file of this document runtime; the core's live in ../../core.""" + here = ROOT / name + return here if here.exists() else ROOT.parent.parent / 'core' / name + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--server', type=Path, default=ROOT / 'build/pure-connections') @@ -532,7 +538,7 @@ def main(): result['environment'] = {'platform': platform.platform(), 'machine': platform.machine(), 'python': platform.python_version(), 'node': subprocess.check_output(['node', '--version'], text=True).strip()} result['source_sha256'] = {str(path.relative_to(ROOT.parent.parent)): hashlib.sha256(path.read_bytes()).hexdigest() - for path in [args.store_source.resolve(), *(ROOT / name for name in ('flow.c', 'flow.h', 'store.h', 'program.c', 'program.h', + for path in [args.store_source.resolve(), *(source(name) for name in ('flow.c', 'flow.h', 'store.h', 'program.c', 'program.h', 'runtime.c', 'runtime.h', 'result.h', 'evaluate.c', 'store-port.c', 'store-port.h', 'test-runtime.c', 'server.c', 'check.py', 'check_edits.py', 'edit.c', 'test-splice.c', 'test-flow.c', 'test-program.c', 'test-fields.c', 'Makefile', diff --git a/prototypes/connections/deliver.c b/prototypes/connections/deliver.c deleted file mode 100644 index 0d3da64..0000000 --- a/prototypes/connections/deliver.c +++ /dev/null @@ -1,716 +0,0 @@ -#define _POSIX_C_SOURCE 200809L -#include "deliver.h" -#include "message-json.h" -#include "module-text.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { TIMEOUT_S = 10, FIRST_PAUSE_MS = 100, LONGEST_PAUSE_MS = 10000, GROUPS = PURE_MODULE_NAME / 2, BATCH_MOST = 256, - ANSWER_MOST = 1 << 20 }; -enum { DELIVERED, AGAIN, REFUSED_BY_PEER, TLS_FAILED, ANSWER_UNREAD }; - -typedef struct { char *body; size_t size; uint64_t seq; unsigned char *sent; size_t sent_size; } Pending; -/* An answer made into its input's message: a table of the fields' starts and - * sizes (u32le), then the payload from its prefix on. */ -typedef struct { uint64_t seq; unsigned char *bytes; size_t size, table; } Answer; - -struct PureWire { - const PureModule *module; - uint32_t output; - char host[256], port[8], path[1024], source[PURE_SOURCE + 1]; - pthread_mutex_t lock; - pthread_cond_t changed; - Pending *queue; /* the sends to deliver, in order, from head */ - size_t head, count, capacity; - uint64_t queued, delivered, attempts, deliveries; - unsigned batch; /* how many a request may carry, as the receiver said (X-Pure-Batch); 0: one */ - bool stopping, started, stuck; - pthread_t thread; - void (*notify)(void *); - void *context; - int fd; /* the connection kept for the next send, or -1; changed under the lock */ - bool tls; /* https: the connection is TLS */ - SSL *ssl; /* its TLS session */ - int answer_input; /* the input answers become events of, or -1 */ - Answer *answers; /* made, for the owner to take in order */ - size_t answer_head, answer_count, answer_capacity; - Answer taken; /* the one the owner took last, kept until it takes the next */ - uint32_t placed[2 * PURE_MODULE_FIELDS]; -}; - -/* ---- TLS: one client context, verifying the receiver's certificate against - * the system's CAs or the file pure_wire_trust named, and its name. */ -static pthread_once_t tls_once = PTHREAD_ONCE_INIT; -static SSL_CTX *tls_context; -static const char *tls_ca; - -void pure_wire_trust(const char *ca_file) { tls_ca = ca_file; } - -static void make_tls_context(void) { - SSL_CTX *context = SSL_CTX_new(TLS_client_method()); - if (!context) return; - if (!SSL_CTX_set_min_proto_version(context, TLS1_2_VERSION) || - !(tls_ca ? SSL_CTX_load_verify_locations(context, tls_ca, NULL) : SSL_CTX_set_default_verify_paths(context))) { - SSL_CTX_free(context); - return; - } - SSL_CTX_set_verify(context, SSL_VERIFY_PEER, NULL); - SSL_CTX_set_mode(context, SSL_MODE_AUTO_RETRY); - tls_context = context; -} - -/* ---- A send as JSON: its fields in order, a group's fields as an object. */ - -typedef struct { char *bytes; size_t size, capacity; bool failed; } Text; - -static void put(Text *t, const void *bytes, size_t size) { - if (t->failed) return; - if (size > t->capacity - t->size) { - size_t capacity = t->capacity ? t->capacity : 256; - while (size > capacity - t->size) capacity *= 2; - char *grown = realloc(t->bytes, capacity); - if (!grown) { t->failed = true; return; } - t->bytes = grown; - t->capacity = capacity; - } - memcpy(t->bytes + t->size, bytes, size); - t->size += size; -} - -static void put_string(Text *t, const unsigned char *bytes, size_t size) { - put(t, "\"", 1); - size_t run = 0; - for (size_t i = 0; i < size; ++i) { - unsigned char b = bytes[i]; - if (b != '"' && b != '\\' && b >= 0x20) continue; - put(t, bytes + run, i - run); - char escaped[8]; - if (b == '"' || b == '\\') put(t, (char[]){'\\', (char)b}, 2); - else put(t, escaped, (size_t)snprintf(escaped, sizeof(escaped), "\\u%04x", b)); - run = i + 1; - } - put(t, bytes + run, size - run); - put(t, "\"", 1); -} - -static uint32_t read_u32(const unsigned char *p) { return p[0] | p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; } - -/* The group a field is in: the length of its name before the last dot (0: none). */ -static size_t group_of(const char *name) { - const char *dot = strrchr(name, '.'); - return dot ? (size_t)(dot - name) : 0; -} - -/* Whether the group path open (length open_size) holds the group of name (length size). */ -static bool holds(const char *open, size_t open_size, const char *name, size_t size) { - return open_size <= size && !memcmp(open, name, open_size) && (open_size == size || !open_size || name[open_size] == '.'); -} - -static bool encode(const PureWire *w, PureSpan payload, Text *t) { - const PureModule *m = w->module; - const PureInputShape *shape = &m->input[w->output]; - if (payload.size < shape->prefix) return false; - char open[PURE_MODULE_NAME] = ""; - size_t open_size = 0, at = shape->prefix; - unsigned depth = 0; - bool first[GROUPS + 1] = {true}; - put(t, "{", 1); - for (uint32_t f = 0; f < shape->field_count; ++f) { - const PureField *field = &m->field[shape->field_start + f]; - size_t group = group_of(field->name); - while (depth && !holds(open, open_size, field->name, group)) { - put(t, "}", 1); - --depth; - open_size = group_of(open) < open_size ? group_of(open) : 0; - open[open_size] = 0; - } - while (open_size < group) { - size_t start = open_size ? open_size + 1 : 0, end = start; - while (end < group && field->name[end] != '.') ++end; - if (!first[depth]) put(t, ",", 1); - first[depth] = false; - put_string(t, (const unsigned char *)field->name + start, end - start); - put(t, ":{", 2); - first[++depth] = true; - memcpy(open, field->name, end); - open[open_size = end] = 0; - } - if (!first[depth]) put(t, ",", 1); - first[depth] = false; - const char *key = field->name + (group ? group + 1 : 0); - put_string(t, (const unsigned char *)key, strlen(key)); - put(t, ":", 1); - char digits[32]; - if (field->type == PURE_FIELD_U32) { - put(t, digits, (size_t)snprintf(digits, sizeof(digits), "%u", (unsigned)read_u32(payload.data + field->offset))); - } else if (field->type == PURE_FIELD_NUMBER) { - uint64_t bits = read_u32(payload.data + field->offset) | (uint64_t)read_u32(payload.data + field->offset + 4) << 32; - double value; - memcpy(&value, &bits, sizeof(value)); - put(t, digits, pure_format_number(value, digits)); - } else { - size_t size = field->offset == PURE_FIELD_REST ? payload.size - at : read_u32(payload.data + field->offset); - if (size > payload.size - at) return false; - PureSpan value = {payload.data + at, size}; - if (!pure_utf8_valid(value)) return false; /* bytes that are not text have no JSON form */ - put_string(t, value.data, size); - at += size; - } - } - while (depth--) put(t, "}", 1); - put(t, "}", 1); - return !t->failed; -} - -/* ---- The wire. */ - -PureWire *pure_wire_open(const PureModule *m, const char *output, const char *url, const char *source, - char *error, size_t size) { - int index = pure_module_input(m, output); - if (index < 0 || m->input[index].kind != PURE_KIND_OUTPUT) { - snprintf(error, size, "no output '%s'", output); - return NULL; - } - /* A group's fields are together, so each group is one object. */ - const PureInputShape *shape = &m->input[index]; - for (uint32_t f = 1; f < shape->field_count; ++f) { - const char *name = m->field[shape->field_start + f].name; - for (uint32_t e = 0; e + 1 < f; ++e) { - const char *earlier = m->field[shape->field_start + e].name, *between = m->field[shape->field_start + f - 1].name; - size_t shared = 0; - while (earlier[shared] && earlier[shared] == name[shared]) ++shared; - while (shared && earlier[shared - 1] != '.') --shared; /* the groups both are in */ - if (shared && strncmp(between, name, shared)) { - snprintf(error, size, "the fields of output '%s' in one group are not together", output); - return NULL; - } - } - } - size_t length = strlen(source); - if (!length || length > PURE_SOURCE) { - snprintf(error, size, "a source name has 1 to %d bytes", PURE_SOURCE); - return NULL; - } - bool tls = !strncmp(url, "https://", 8); - if (!tls && strncmp(url, "http://", 7)) { - snprintf(error, size, "a URL is http[s]://host[:port]/path"); - return NULL; - } - if (tls) { - pthread_once(&tls_once, make_tls_context); - if (!tls_context) { - snprintf(error, size, "no TLS context: %s cannot be read", tls_ca ? tls_ca : "the system's CAs"); - return NULL; - } - } - const char *host = url + (tls ? 8 : 7), *slash = strchr(host, '/'), *colon = memchr(host, ':', slash ? (size_t)(slash - host) : strlen(host)); - const char *host_end = colon ? colon : slash ? slash : host + strlen(host); - const char *port = colon ? colon + 1 : tls ? "443" : "80"; - size_t port_size = colon ? (size_t)((slash ? slash : host + strlen(host)) - port) : strlen(port); - if (host_end == host || (size_t)(host_end - host) >= 256 || !port_size || port_size >= 8 || - (slash && strlen(slash) >= 1024)) { - snprintf(error, size, "a URL is http://host[:port]/path"); - return NULL; - } - PureWire *w = calloc(1, sizeof(*w)); - if (!w) { snprintf(error, size, "out of memory"); return NULL; } - w->module = m; - w->output = (uint32_t)index; - memcpy(w->host, host, (size_t)(host_end - host)); - memcpy(w->port, port, port_size); - snprintf(w->path, sizeof(w->path), "%s", slash ? slash : "/"); - memcpy(w->source, source, length + 1); - w->fd = -1; - w->tls = tls; - w->answer_input = -1; - pthread_mutex_init(&w->lock, NULL); - pthread_cond_init(&w->changed, NULL); - return w; -} - -uint32_t pure_wire_output(const PureWire *w) { return w->output; } - -bool pure_wire_reply(PureWire *w, uint32_t input, char *error, size_t size) { - const PureModule *m = w->module; - if (input >= m->inputs || m->input[input].kind != PURE_KIND_MESSAGE || m->input[input].keyed) { - snprintf(error, size, "an answer goes to a message input without a key"); - return false; - } - w->answer_input = (int)input; - return true; -} - -bool pure_wire_answer(PureWire *w, PureWireAnswer *out) { - free(w->taken.bytes); - w->taken = (Answer){0}; - pthread_mutex_lock(&w->lock); - bool some = w->answer_count > 0; - if (some) { - w->taken = w->answers[w->answer_head++]; - --w->answer_count; - w->delivered = w->taken.seq; /* trimmed after the owner applies it */ - } - pthread_mutex_unlock(&w->lock); - if (!some) return false; - const PureInputShape *in = &w->module->input[w->answer_input]; - for (size_t i = 0; i < 2 * (size_t)in->field_count; ++i) w->placed[i] = read_u32(w->taken.bytes + 4 * i); - *out = (PureWireAnswer){(uint32_t)w->answer_input, w->taken.seq, - {w->taken.bytes + w->taken.table, w->taken.size - w->taken.table}, w->placed, w->source}; - return true; -} -uint64_t pure_wire_queued(const PureWire *w) { return w->queued; } - -uint64_t pure_wire_delivered(PureWire *w) { - pthread_mutex_lock(&w->lock); - uint64_t delivered = w->delivered; - pthread_mutex_unlock(&w->lock); - return delivered; -} - -void pure_wire_counts(PureWire *w, uint64_t *delivered, uint64_t *attempts) { - pthread_mutex_lock(&w->lock); - *delivered = w->deliveries; - *attempts = w->attempts; - pthread_mutex_unlock(&w->lock); -} - -bool pure_wire_queue(PureWire *w, uint64_t seq, PureSpan payload) { - if (w->stuck) return false; - Text t = {0}; - if (!encode(w, payload, &t)) { - free(t.bytes); - w->stuck = true; - fprintf(stderr, "{\"wire\":\"%s\",\"seq\":%llu,\"error\":\"no JSON form: its bytes are not text\"}\n", w->source, - (unsigned long long)seq); - return false; - } - pthread_mutex_lock(&w->lock); - if (w->head + w->count == w->capacity) { - if (w->head) { - memmove(w->queue, w->queue + w->head, w->count * sizeof(*w->queue)); - w->head = 0; - } else { - size_t capacity = w->capacity ? 2 * w->capacity : 16; - Pending *grown = realloc(w->queue, capacity * sizeof(*grown)); - if (!grown) { - pthread_mutex_unlock(&w->lock); - free(t.bytes); - return false; /* it stays in the outbox: queued after the next save */ - } - w->queue = grown; - w->capacity = capacity; - } - } - unsigned char *sent = NULL; - if (w->answer_input >= 0 && !(sent = malloc(payload.size ? payload.size : 1))) { - pthread_mutex_unlock(&w->lock); - free(t.bytes); - return false; - } - if (sent && payload.size) memcpy(sent, payload.data, payload.size); - w->queue[w->head + w->count++] = (Pending){t.bytes, t.size, seq, sent, payload.size}; - w->queued = seq; - pthread_cond_signal(&w->changed); - pthread_mutex_unlock(&w->lock); - return true; -} - -/* ---- HTTP/1.1, one request at a time on a kept connection. */ - -typedef struct { PureWire *wire; char bytes[4096]; size_t at, end; } Reader; - -static bool fill(Reader *r) { - if (r->at == r->end) r->at = r->end = 0; - if (r->end == sizeof(r->bytes)) return false; - PureWire *w = r->wire; - ssize_t n; - if (w->ssl) n = SSL_read(w->ssl, r->bytes + r->end, (int)(sizeof(r->bytes) - r->end)); - else do n = recv(w->fd, r->bytes + r->end, sizeof(r->bytes) - r->end, 0); while (n < 0 && errno == EINTR); - if (n <= 0) return false; -#ifdef TCP_QUICKACK - /* Acknowledge at once: a server that writes an answer in pieces holds the - * next piece until the first is acknowledged (Nagle), which a delayed - * acknowledgement would stall for 40 ms. The flag does not stay set. */ - int one = 1; - setsockopt(w->fd, IPPROTO_TCP, TCP_QUICKACK, &one, sizeof(one)); -#endif - r->end += (size_t)n; - return true; -} - -/* A line without its CRLF, in line (at most size - 1 bytes). */ -static bool read_line(Reader *r, char *line, size_t size) { - for (;;) { - char *end = memchr(r->bytes + r->at, '\n', r->end - r->at); - if (end) { - size_t length = (size_t)(end - (r->bytes + r->at)); - if (length && end[-1] == '\r') --length; - if (length >= size) return false; - memcpy(line, r->bytes + r->at, length); - line[length] = 0; - r->at = (size_t)(end - r->bytes) + 1; - return true; - } - if (r->at) { - memmove(r->bytes, r->bytes + r->at, r->end - r->at); - r->end -= r->at; - r->at = 0; - } - if (!fill(r)) return false; - } -} - -/* size bytes of the body: kept in into (up to ANSWER_MOST), or skipped when it is NULL. */ -static bool take(Reader *r, uint64_t size, Text *into) { - if (into && size > ANSWER_MOST - into->size) return false; - while (size) { - if (r->at == r->end && !fill(r)) return false; - size_t n = r->end - r->at < size ? r->end - r->at : (size_t)size; - if (into) put(into, r->bytes + r->at, n); - r->at += n; - size -= n; - } - return !into || !into->failed; -} - -static bool send_all(PureWire *w, const char *bytes, size_t size) { - while (size) { - ssize_t n = w->ssl ? SSL_write(w->ssl, bytes, (int)(size < INT32_MAX ? size : INT32_MAX)) : - send(w->fd, bytes, size, MSG_NOSIGNAL); - if (n < 0 && !w->ssl && errno == EINTR) continue; - if (n <= 0) return false; - bytes += n; - size -= (size_t)n; - } - return true; -} - -/* TLS over a connected socket: the receiver's certificate must name the host - * (a DNS name, or an IP address written as one). */ -static SSL *start_tls(const PureWire *w, int fd) { - SSL *ssl = SSL_new(tls_context); - if (!ssl) return NULL; - unsigned char address[16]; - bool literal = inet_pton(AF_INET, w->host, address) == 1 || inet_pton(AF_INET6, w->host, address) == 1; - bool named = literal ? X509_VERIFY_PARAM_set1_ip_asc(SSL_get0_param(ssl), w->host) == 1 : - SSL_set_tlsext_host_name(ssl, w->host) == 1 && SSL_set1_host(ssl, w->host) == 1; - if (!named || SSL_set_fd(ssl, fd) != 1 || SSL_connect(ssl) != 1 || SSL_get_verify_result(ssl) != X509_V_OK) { - SSL_free(ssl); - return NULL; - } - return ssl; -} - -static int connect_to(const PureWire *w) { - struct addrinfo hints = {.ai_family = AF_UNSPEC, .ai_socktype = SOCK_STREAM}, *list; - if (getaddrinfo(w->host, w->port, &hints, &list)) return -1; - int fd = -1; - for (struct addrinfo *a = list; a && fd < 0; a = a->ai_next) { - fd = socket(a->ai_family, a->ai_socktype | SOCK_CLOEXEC, a->ai_protocol); - if (fd < 0) continue; - struct timeval limit = {TIMEOUT_S, 0}; - int one = 1; - setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &limit, sizeof(limit)); - setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &limit, sizeof(limit)); - setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); - if (connect(fd, a->ai_addr, a->ai_addrlen)) { - close(fd); - fd = -1; - } - } - freeaddrinfo(list); - return fd; -} - -static void drop_connection(PureWire *w) { - pthread_mutex_lock(&w->lock); - if (w->ssl) SSL_free(w->ssl); - if (w->fd >= 0) close(w->fd); - w->ssl = NULL; - w->fd = -1; - pthread_mutex_unlock(&w->lock); -} - -/* A fault a Pure receiver answers when its app refused the message. */ -static bool refusal(const char *error) { - static const char *const faults[] = {"divide", "non_finite", "clamp", "limit", "not_found", "encoding", "exists"}; - for (size_t i = 0; i < sizeof(faults) / sizeof(*faults); ++i) if (!strcmp(error, faults[i])) return true; - return false; -} - -/* The answer to one request: its status, whether a Pure receiver refused it, - * and whether the connection may carry the next. */ -static bool read_answer(PureWire *w, int *status, bool *refused, bool *keep, unsigned *batch, Text *body) { - Reader r = {.wire = w}; - char line[1024]; - if (!read_line(&r, line, sizeof(line)) || strncmp(line, "HTTP/1.", 7) || strlen(line) < 12) return false; - *status = atoi(line + 9); - *keep = line[7] == '1'; - *refused = false; - uint64_t length = 0; - bool sized = false, chunked = false; - while (read_line(&r, line, sizeof(line))) { - if (!*line) { - if (chunked) { - for (;;) { - uint64_t chunk; - if (!read_line(&r, line, sizeof(line))) return false; - chunk = strtoull(line, NULL, 16); - if (!chunk) break; - if (!take(&r, chunk, body) || !read_line(&r, line, sizeof(line))) return false; - } - while (read_line(&r, line, sizeof(line)) && *line) {} - return true; - } - if (*status == 204 || *status == 304 || *status < 200) return true; - if (sized) return take(&r, length, body); - *keep = false; /* the body runs to the close */ - while (fill(&r)) { - if (body && !take(&r, r.end - r.at, body)) return false; - r.at = r.end; - } - return true; - } - char *value = strchr(line, ':'); - if (!value) return false; - *value++ = 0; - while (*value == ' ' || *value == '\t') ++value; - if (!strcasecmp(line, "Content-Length")) { length = strtoull(value, NULL, 10); sized = true; } - else if (!strcasecmp(line, "Transfer-Encoding")) chunked = !strcasecmp(value, "chunked"); - else if (!strcasecmp(line, "Connection")) *keep = strcasecmp(value, "close") != 0 && *keep; - else if (!strcasecmp(line, "X-Pure-Error")) *refused = refusal(value); - else if (!strcasecmp(line, "X-Pure-Batch")) *batch = (unsigned)strtoul(value, NULL, 10); - } - return false; -} - -static void write_u32(unsigned char *p, uint32_t v) { for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(v >> (8 * i)); } - -/* The answer (status, body) to the send p as a message of the answer input: - * fields from the body's JSON object by name, others skipped; a field the - * body lacks from the send's field of that name and type; a u32 field named - * http_status from the status. False when a field has none of these. */ -static bool form_answer(const PureWire *w, int status, const Text *body, const Pending *p, Answer *out) { - const PureModule *m = w->module; - const PureInputShape *in = &m->input[w->answer_input], *sent = &m->input[w->output]; - size_t table = 8 * (size_t)in->field_count, total = table + in->prefix + body->size + p->sent_size; - unsigned char *bytes = calloc(1, total ? total : 1); - if (!bytes) return false; - unsigned char *prefix = bytes + table, *text = prefix + in->prefix, *copy = text + body->size; - if (body->size) memcpy(text, body->bytes, body->size); - if (p->sent_size) memcpy(copy, p->sent, p->sent_size); - PureJsonMessage message = {.module = m, .shape = in, .payload = prefix, .table = bytes, .lenient = true}; - size_t at = pure_json_space(text, 0, body->size); - if (at < body->size && text[at] == '{' && pure_json_message(&message, text, body->size, &at)) { - free(bytes); - return false; - } - for (uint32_t f = 0; f < in->field_count; ++f) { - if (message.seen[f]) continue; - const PureField *field = &m->field[in->field_start + f]; - if (field->type == PURE_FIELD_U32 && !strcmp(field->name, "http_status")) { - write_u32(prefix + field->offset, (uint32_t)status); - continue; - } - /* The send's field of the same name and type, and where its bytes are. */ - size_t start = sent->prefix, length = 0; - const PureField *from = NULL; - for (uint32_t g = 0; g < sent->field_count && !from; ++g) { - const PureField *other = &m->field[sent->field_start + g]; - bool variable = other->type == PURE_FIELD_BYTES || other->type == PURE_FIELD_TEXT; - if (variable) length = other->offset == PURE_FIELD_REST ? p->sent_size - start : read_u32(p->sent + other->offset); - if (!strcmp(other->name, field->name) && other->type == field->type) from = other; - else if (variable) start += length; - } - bool variable = field->type == PURE_FIELD_BYTES || field->type == PURE_FIELD_TEXT; - if (!from || (variable && (start > p->sent_size || length > p->sent_size - start))) { - free(bytes); - return false; - } - if (field->type == PURE_FIELD_U32 || field->type == PURE_FIELD_NUMBER) - memcpy(prefix + field->offset, p->sent + from->offset, field->type == PURE_FIELD_U32 ? 4 : 8); - else { - write_u32(bytes + 8 * f, (uint32_t)(copy + start - prefix)); - write_u32(bytes + 8 * f + 4, (uint32_t)length); - } - } - *out = (Answer){p->seq, bytes, total, table}; - return true; -} - -/* One request for count sends from p (numbered in a row): one object, or - * an array of them for a receiver that takes batches. A final answer to a - * send whose answers are wired is made into *answer. */ -static int post(PureWire *w, const Pending *p, unsigned count, Answer *answer) { - size_t body = count > 1 ? count + 1 : 0; - for (unsigned i = 0; i < count; ++i) body += p[i].size; - char head[2048], batch[32] = ""; - if (count > 1) snprintf(batch, sizeof(batch), "X-Pure-Count: %u\r\n", count); - int size = snprintf(head, sizeof(head), - "POST %s HTTP/1.1\r\nHost: %s:%s\r\nContent-Type: application/json\r\nContent-Length: %zu\r\n" - "X-Pure-Source: %s\r\nX-Pure-Sequence: %llu\r\n%sIdempotency-Key: %s-%llu\r\n\r\n", - w->path, w->host, w->port, body, w->source, (unsigned long long)p->seq, batch, w->source, - (unsigned long long)p->seq); - /* A kept connection the peer closed is tried once more on a new one. */ - for (int attempt = 0; attempt < 2; ++attempt) { - bool fresh = w->fd < 0; - if (fresh) { - int fd = connect_to(w); - if (fd < 0) return AGAIN; - pthread_mutex_lock(&w->lock); - w->fd = fd; - bool stopping = w->stopping; - pthread_mutex_unlock(&w->lock); - if (stopping) return AGAIN; - SSL *ssl = w->tls ? start_tls(w, fd) : NULL; - if (w->tls && !ssl) { - drop_connection(w); - return TLS_FAILED; - } - pthread_mutex_lock(&w->lock); - w->ssl = ssl; - pthread_mutex_unlock(&w->lock); - } - int status; - bool refused, keep, sent = send_all(w, head, (size_t)size); - unsigned batch_size = 0; - Text body = {0}; - for (unsigned i = 0; sent && i < count; ++i) - sent = (!i || count == 1 || send_all(w, ",", 1)) && (i || count == 1 || send_all(w, "[", 1)) && - send_all(w, p[i].body, p[i].size); - if (sent && count > 1) sent = send_all(w, "]", 1); - if (!sent || !read_answer(w, &status, &refused, &keep, &batch_size, w->answer_input >= 0 ? &body : NULL)) { - free(body.bytes); - drop_connection(w); - if (fresh) return AGAIN; - continue; - } - if (!keep) drop_connection(w); - if (batch_size && w->answer_input < 0) w->batch = batch_size; - if (status == 408 || status == 429 || status >= 500) { - free(body.bytes); - return AGAIN; - } - if (w->answer_input >= 0 && status >= 200 && status < 500) { /* a final answer: the app hears it */ - bool formed = form_answer(w, status, &body, p, answer); - free(body.bytes); - return formed ? DELIVERED : ANSWER_UNREAD; - } - free(body.bytes); - if ((status >= 200 && status < 300) || (status >= 400 && status < 500 && refused)) return DELIVERED; - return REFUSED_BY_PEER; - } - return AGAIN; -} - -static void *deliver(void *context) { - PureWire *w = context; - long pause_ms = 0; - uint64_t reported = 0; - pthread_mutex_lock(&w->lock); - for (;;) { - while (!w->stopping && !w->count) pthread_cond_wait(&w->changed, &w->lock); - if (w->stopping) break; - /* As many as the receiver takes at once; their bodies stay put while - * the queue only grows behind them. */ - unsigned count = w->batch > 1 ? (unsigned)(w->count < w->batch ? w->count : w->batch) : 1; - Pending next[BATCH_MOST]; - if (count > BATCH_MOST) count = BATCH_MOST; - memcpy(next, w->queue + w->head, count * sizeof(*next)); - ++w->attempts; - pthread_mutex_unlock(&w->lock); - Answer answer = {0}; - int outcome = post(w, next, count, &answer); - pthread_mutex_lock(&w->lock); - if (outcome == DELIVERED && answer.bytes) { /* for the owner, who then counts it delivered */ - if (w->answer_head + w->answer_count == w->answer_capacity) { - if (w->answer_head) { - memmove(w->answers, w->answers + w->answer_head, w->answer_count * sizeof(*w->answers)); - w->answer_head = 0; - } else { - size_t capacity = w->answer_capacity ? 2 * w->answer_capacity : 16; - Answer *grown = realloc(w->answers, capacity * sizeof(*grown)); - if (!grown) { free(answer.bytes); outcome = AGAIN; } - else { w->answers = grown; w->answer_capacity = capacity; } - } - } - if (outcome == DELIVERED) w->answers[w->answer_head + w->answer_count++] = answer; - } - if (outcome == DELIVERED) { - for (unsigned i = 0; i < count; ++i) { - free(w->queue[w->head + i].body); - free(w->queue[w->head + i].sent); - } - w->head += count; - w->count -= count; - if (!answer.bytes) w->delivered = next[count - 1].seq; - w->deliveries += count; - pause_ms = 0; - pthread_mutex_unlock(&w->lock); - if (w->notify) w->notify(w->context); - pthread_mutex_lock(&w->lock); - continue; - } - if (outcome != AGAIN && reported != next[0].seq) { - fprintf(stderr, "{\"wire\":\"%s\",\"seq\":%llu,\"error\":\"%s\"}\n", w->source, - (unsigned long long)next[0].seq, outcome == TLS_FAILED ? "the receiver's certificate is not trusted; trying again" : - outcome == ANSWER_UNREAD ? "its answer does not make the answer input's message; trying again" : - "the receiver does not take it; trying again"); - reported = next[0].seq; - } - pause_ms = outcome != AGAIN ? LONGEST_PAUSE_MS : pause_ms ? 2 * pause_ms : FIRST_PAUSE_MS; - if (pause_ms > LONGEST_PAUSE_MS) pause_ms = LONGEST_PAUSE_MS; - struct timespec until; - clock_gettime(CLOCK_REALTIME, &until); - until.tv_sec += pause_ms / 1000; - until.tv_nsec += (pause_ms % 1000) * 1000000; - if (until.tv_nsec >= 1000000000) { ++until.tv_sec; until.tv_nsec -= 1000000000; } - while (!w->stopping && pthread_cond_timedwait(&w->changed, &w->lock, &until) == 0) {} - } - pthread_mutex_unlock(&w->lock); - return NULL; -} - -bool pure_wire_start(PureWire *w, void (*notify)(void *), void *context) { - w->notify = notify; - w->context = context; - w->started = pthread_create(&w->thread, NULL, deliver, w) == 0; - return w->started; -} - -void pure_wire_close(PureWire *w) { - if (!w) return; - pthread_mutex_lock(&w->lock); - w->stopping = true; - if (w->fd >= 0) shutdown(w->fd, SHUT_RDWR); /* a send or an answer waited for ends now */ - pthread_cond_broadcast(&w->changed); - pthread_mutex_unlock(&w->lock); - if (w->started) pthread_join(w->thread, NULL); - if (w->ssl) SSL_free(w->ssl); - if (w->fd >= 0) close(w->fd); - for (size_t i = 0; i < w->count; ++i) { - free(w->queue[w->head + i].body); - free(w->queue[w->head + i].sent); - } - for (size_t i = 0; i < w->answer_count; ++i) free(w->answers[w->answer_head + i].bytes); - free(w->answers); - free(w->taken.bytes); - free(w->queue); - pthread_cond_destroy(&w->changed); - pthread_mutex_destroy(&w->lock); - free(w); -} diff --git a/prototypes/connections/deliver.h b/prototypes/connections/deliver.h deleted file mode 100644 index 2d1da29..0000000 --- a/prototypes/connections/deliver.h +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef PURE_DELIVER_H -#define PURE_DELIVER_H -#include "module.h" - -/* Delivering an app's saved sends over HTTP (docs/research/083-delivery-over-http.md). - * A wire takes one output's sends in order: each goes as a JSON object of - * its fields (groups as objects inside) by POST to a URL, with the headers - * X-Pure-Source (the sender's name for the output), X-Pure-Sequence and - * Idempotency-Key, one at a time, again and again until it is delivered: - * a 2xx answer, or a Pure receiver's refusal (a 4xx with X-Pure-Error - * naming a fault: it heard the message and refused it). 408, 429, 5xx and - * no answer are tried again after a pause (0.1 s doubling to 10 s); another - * 4xx is logged and tried again every 10 s. Nothing is ever dropped: what is - * not delivered stays in the app's outbox, and a receiver applies a number - * once, so a send delivered twice (its answer lost) is harmless. */ -typedef struct PureWire PureWire; - -/* A wire for the output named output of m to url (http[s]://host[:port]/path), - * as source; NULL with why in error when it cannot be. An https receiver's - * certificate must be trusted (the system's CAs, or the file given to - * pure_wire_trust before the first https wire) and name its host. */ -void pure_wire_trust(const char *ca_file); -PureWire *pure_wire_open(const PureModule *m, const char *output, const char *url, const char *source, - char *error, size_t size); -uint32_t pure_wire_output(const PureWire *); -/* The owner's side: queue a saved send numbered after the last queued one - * (it is encoded now); false when it cannot be (a bytes field that is not - * UTF-8 has no JSON form), and the wire then stops at it. */ -bool pure_wire_queue(PureWire *, uint64_t seq, PureSpan payload); -uint64_t pure_wire_queued(const PureWire *); -/* The last number delivered; any thread. */ -uint64_t pure_wire_delivered(PureWire *); -/* Deliver on a thread of the wire's own, calling notify after each delivery. */ -bool pure_wire_start(PureWire *, void (*notify)(void *), void *context); -/* Stop (a delivery under way is finished or abandoned) and free. */ -void pure_wire_close(PureWire *); -/* Answers as events: the final answer to each send (a 2xx, or a 4xx but 408 - * and 429) becomes a message event of input (a message without a key), in - * the order sent: its fields come from the answer's JSON object by name - * (other members are skipped), a field the answer lacks from the send's - * field of that name and type (as a request's id), and a u32 field named - * http_status is the answer's status. An answer that makes no message is - * logged and the send tried again every 10 s. Answers go one per request. */ -bool pure_wire_reply(PureWire *, uint32_t input, char *error, size_t size); -/* The owner's side: the next answer in order, or false. Taking it counts - * its send delivered (the owner applies the answer, then trims); what out - * points to is the wire's until the next call. The answer is applied with - * pure_app_deliver from the wire's source with the send's number, so an - * answer that comes again after a restart is applied once. */ -typedef struct { - uint32_t input; - uint64_t seq; - PureSpan payload; - const uint32_t *placed; - const char *source; -} PureWireAnswer; -bool pure_wire_answer(PureWire *, PureWireAnswer *out); -/* Counts for a host's report. */ -void pure_wire_counts(PureWire *, uint64_t *delivered, uint64_t *attempts); -#endif diff --git a/prototypes/connections/edit-module.c b/prototypes/connections/edit-module.c deleted file mode 100644 index 3ce272e..0000000 --- a/prototypes/connections/edit-module.c +++ /dev/null @@ -1,179 +0,0 @@ -#define _POSIX_C_SOURCE 200809L -#include "module-store.h" -#include "store.h" -#include -#include -#include -#include - -/* The document CLI contract of edit.c, driven by a verified Pure module - * instead of a document program. The module declares keyed messages named - * show and edit; edit has u32 fields start and remove and a bytes tail. */ -static bool number(const char *text, uint64_t limit, uint64_t *out) { - if (!*text) return false; - uint64_t value = 0; - for (; *text; ++text) { - if (*text < '0' || *text > '9') return false; - unsigned digit = (unsigned)(*text - '0'); - if (digit > limit || value > (limit - digit) / 10) return false; - value = value * 10 + digit; - } - *out = value; - return true; -} - -static const char *message(PureResult status) { - switch (status) { - case PURE_OK: return "ok"; - case PURE_NOT_FOUND: return "not_found"; - case PURE_CONFLICT: return "conflict"; - case PURE_BUSY: return "busy"; - case PURE_BUDGET: return "payload_budget"; - case PURE_LIMIT: return "limit"; - case PURE_UNCERTAIN: return "durability_unknown"; - case PURE_DENIED: return "denied"; - case PURE_ALLOCATION: return "allocation"; - case PURE_INVALID: return "invalid"; - default: return "failed"; - } -} - -static bool valid_key(const char *key) { - size_t length = strlen(key); - if (!length || length > 64) return false; - for (size_t i = 0; i < length; ++i) { - char c = key[i]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || - c == '-' || c == '_')) return false; - } - return true; -} - -static void put32(unsigned char *to, uint32_t value) { - for (unsigned i = 0; i < 4; ++i) to[i] = (unsigned char)(value >> (8 * i)); -} - -static int field(const PureModule *m, int input, const char *name, uint8_t type) { - const PureInputShape *shape = &m->input[input]; - for (uint32_t i = 0; i < shape->field_count; ++i) - if (!strcmp(m->field[shape->field_start + i].name, name) && m->field[shape->field_start + i].type == type) - return (int)(shape->field_start + i); - return -1; -} - -static bool write_span(void *context, PureSpan span) { - return fwrite(span.data, 1, span.size, context) == span.size; -} - -static bool count_span(void *context, PureSpan span) { - *(size_t *)context += span.size; - return true; -} - -int main(int argc, char **argv) { - const char *module_path = NULL, *store_path = NULL; - size_t budget = 4u * 1024u * 1024u; - int at = 1; - for (; at + 1 < argc && !strncmp(argv[at], "--", 2); at += 2) { - uint64_t value; - if (!strcmp(argv[at], "--store")) store_path = argv[at + 1]; - else if (!strcmp(argv[at], "--module")) module_path = argv[at + 1]; - else if (!strcmp(argv[at], "--budget") && number(argv[at + 1], SIZE_MAX, &value)) budget = (size_t)value; - else goto usage; - } - if (!module_path || !store_path || at >= argc) goto usage; - bool show = !strcmp(argv[at], "show"), edit = !strcmp(argv[at], "edit"); - if ((!show && !edit && strcmp(argv[at], "apply")) || - argc - at != (show ? 2 : edit ? 6 : 4)) goto usage; - uint64_t expected = 0, offset = 0, remove = 0; - if (!show && !number(argv[at + 2], INT64_MAX, &expected)) goto usage; - if (edit && (!number(argv[at + 3], UINT32_MAX, &offset) || !number(argv[at + 4], UINT32_MAX, &remove))) goto usage; - - static PureModule module; - char error[160]; - PureResult read = pure_module_read(module_path, &module, error, sizeof(error)); - if (read != PURE_OK) { - fprintf(stderr, "Module: %s\n", read == PURE_NOT_FOUND ? "cannot read module" : error); - return 2; - } - int input = pure_module_input(&module, show ? "show" : "edit"); - if (input < 0 || module.input[input].kind != PURE_KIND_MESSAGE || !module.input[input].keyed) { - fprintf(stderr, "Module: needs a keyed message named %s\n", show ? "show" : "edit"); - return 2; - } - const PureInputShape *shape = &module.input[input]; - int start_field = field(&module, input, "start", PURE_FIELD_U32); - int remove_field = field(&module, input, "remove", PURE_FIELD_U32); - size_t prefix = edit ? shape->prefix : 0; - if (edit && (start_field < 0 || remove_field < 0 || !shape->field_count || - module.field[shape->field_start + shape->field_count - 1].type != PURE_FIELD_BYTES)) { - fputs("edit requires u32 fields start and remove and trailing insertion bytes; use apply for a custom input\n", stderr); - return 2; - } - FILE *file = NULL; - size_t length = 0; - if (!show) { - file = fopen(argv[at + (edit ? 5 : 3)], "rb"); - if (!file) { perror("input"); return 2; } - long bytes; - if (fseek(file, 0, SEEK_END) || (bytes = ftell(file)) < 0 || - (uintmax_t)bytes > PURE_OBJECT_LIMIT - prefix || fseek(file, 0, SEEK_SET)) { - fputs("input must be a seekable file within the request size limit\n", stderr); - fclose(file); return 2; - } - length = (size_t)bytes; - } - if (pure_store_init(store_path) != PURE_OK) { - if (file) fclose(file); - fputs("cannot initialize store\n", stderr); return 2; - } - const char *key = argv[at + 1]; - if (!valid_key(key) || prefix + length > budget) { - if (file) fclose(file); - fprintf(stderr, "%s\n", valid_key(key) ? "payload_budget" : "invalid"); return 2; - } - unsigned char *payload = NULL; - if (!show) { - payload = calloc(1, prefix + length ? prefix + length : 1); - if (!payload) { fclose(file); fputs("allocation\n", stderr); return 2; } - if (edit) { - put32(payload + module.field[start_field].offset, (uint32_t)offset); - put32(payload + module.field[remove_field].offset, (uint32_t)remove); - } - bool valid = !length || fread(payload + prefix, 1, length, file) == length; - if (fgetc(file) != EOF || ferror(file)) valid = false; - if (fclose(file)) valid = false; - if (!valid) { - free(payload); - fputs("input changed or could not be read; nothing was applied\n", stderr); return 2; - } - } - PureApp app; - if (pure_app_start(&app, &module, pure_tree_data(store_path)) != PURE_OK) { - free(payload); - fputs("allocation\n", stderr); return 2; - } - PureResult status = pure_app_message(&app, (uint32_t)input, key, show ? -1 : (int64_t)expected, - (PureSpan){payload, prefix + length}); - size_t reply = 0; - if (status == PURE_OK) status = pure_app_reply(&app, count_span, &reply); - if (status == PURE_OK && show && reply > budget) { status = PURE_BUDGET; app.revision = 0; } - int exit_code = 0; - if (status != PURE_OK) { - fprintf(stderr, "{\"error\":\"%s\",\"revision\":\"%" PRId64 "\"}\n", message(status), app.revision); - exit_code = status == PURE_CONFLICT ? 3 : 2; - } else if (show) { - if (pure_app_reply(&app, write_span, stdout) != PURE_OK) exit_code = 2; - fprintf(stderr, "{\"revision\":\"%" PRId64 "\",\"bytes\":%zu}\n", app.revision, reply); - } else if (printf("{\"revision\":\"%" PRId64 "\",\"reply_bytes\":%zu}\n", app.revision, reply) < 0) exit_code = 2; - if (fflush(stdout)) exit_code = 2; - pure_app_stop(&app); - free(payload); - return exit_code; -usage: - fputs("pure-edit-module --module FILE --store DIRECTORY [--budget BYTES] show KEY\n" - "pure-edit-module --module FILE --store DIRECTORY [--budget BYTES] edit KEY REVISION BYTE_OFFSET REMOVE_BYTES INSERT_FILE\n" - "pure-edit-module --module FILE --store DIRECTORY [--budget BYTES] apply KEY REVISION INPUT_FILE\n" - "edit uses the module's start and remove fields; apply accepts its raw message bytes.\n", stderr); - return 2; -} diff --git a/prototypes/connections/flow.c b/prototypes/connections/flow.c deleted file mode 100644 index 1070c70..0000000 --- a/prototypes/connections/flow.c +++ /dev/null @@ -1,41 +0,0 @@ -#include "flow.h" -#include - -void pure_transfer_abort(PureTransfer *t) { - if (!t->active) return; - PureRelease release = t->release; - void *owner = t->owner; - *t = (PureTransfer){0}; - if (release) release(owner); -} - -static void advance(PureTransfer *t) { - while (t->active && t->part < 2 && t->offset == t->spans[t->part].size) { - t->part++; - t->offset = 0; - } - if (t->active && t->part == 2) pure_transfer_abort(t); -} - -void pure_transfer_init(PureTransfer *t, PureSpan header, PureSpan body, - void *owner, PureRelease release) { - assert(!t->active); - assert((!header.size || header.data) && (!body.size || body.data)); - *t = (PureTransfer){.spans = {header, body}, .owner = owner, - .release = release, .active = true}; - advance(t); -} - -PureSpan pure_transfer_next(const PureTransfer *t, size_t limit) { - if (!t->active) return (PureSpan){0}; - PureSpan span = t->spans[t->part]; - size_t remaining = span.size - t->offset; - return (PureSpan){span.data + t->offset, remaining < limit ? remaining : limit}; -} - -bool pure_transfer_consume(PureTransfer *t, size_t bytes) { - if (!t->active || bytes > t->spans[t->part].size - t->offset) return false; - t->offset += bytes; - advance(t); - return true; -} diff --git a/prototypes/connections/flow.h b/prototypes/connections/flow.h deleted file mode 100644 index 6e27c58..0000000 --- a/prototypes/connections/flow.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef PURE_FLOW_H -#define PURE_FLOW_H -#include -#include - -typedef struct { const unsigned char *data; size_t size; } PureSpan; -typedef void (*PureRelease)(void *owner); - -/* A borrowed body stays alive until every byte is consumed or the transfer is - * aborted. A successful send() consumes bytes; peer receipt is a separate fact. - * Single owner thread; callbacks must not reenter or reuse this transfer. */ -typedef struct { - PureSpan spans[2]; - size_t part, offset; - void *owner; - PureRelease release; - bool active; -} PureTransfer; - -void pure_transfer_init(PureTransfer *, PureSpan header, PureSpan body, - void *owner, PureRelease); -PureSpan pure_transfer_next(const PureTransfer *, size_t limit); -bool pure_transfer_consume(PureTransfer *, size_t bytes); -void pure_transfer_abort(PureTransfer *); -#endif diff --git a/prototypes/connections/message-json.c b/prototypes/connections/message-json.c deleted file mode 100644 index 66c812d..0000000 --- a/prototypes/connections/message-json.c +++ /dev/null @@ -1,281 +0,0 @@ -#include "message-json.h" -#include "module-text.h" -#include -#include -#include -#include - -static void write_u32(unsigned char *p, uint32_t v) { for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(v >> (8 * i)); } - -static bool decimal(const char *text, uint64_t maximum, uint64_t *value) { - if (!*text) return false; - uint64_t n = 0; - for (; *text; ++text) { - if (*text < '0' || *text > '9') return false; - unsigned digit = (unsigned)(*text - '0'); - if (n > (maximum - digit) / 10) return false; - n = n * 10 + digit; - } - *value = n; - return true; -} - -/* A JSON number's text: -?(0|[1-9][0-9]*)(.[0-9]+)?([eE][+-]?[0-9]+)?, its - * length (0 if there is none). An integer has neither part after it. */ -static size_t json_number(const unsigned char *p, size_t size, bool *integer) { - size_t i = 0; - *integer = true; - if (i < size && p[i] == '-') ++i; - if (i == size || p[i] < '0' || p[i] > '9') return 0; - if (p[i++] != '0') while (i < size && p[i] >= '0' && p[i] <= '9') ++i; - if (i < size && p[i] == '.') { - *integer = false; - if (++i == size || p[i] < '0' || p[i] > '9') return 0; - while (i < size && p[i] >= '0' && p[i] <= '9') ++i; - } - if (i < size && (p[i] == 'e' || p[i] == 'E')) { - *integer = false; - if (++i < size && (p[i] == '+' || p[i] == '-')) ++i; - if (i == size || p[i] < '0' || p[i] > '9') return 0; - while (i < size && p[i] >= '0' && p[i] <= '9') ++i; - } - return i; -} - -/* A fixed field's value from its text (a query value or a JSON number) into - * the prefix: a u32 in range, or a finite number. */ -bool pure_json_fixed(const PureField *field, const unsigned char *text, size_t size, unsigned char *prefix) { - bool integer; - if (!size || json_number(text, size, &integer) != size || size > 40) return false; - char digits[48]; - memcpy(digits, text, size); - digits[size] = 0; - if (field->type == PURE_FIELD_U32) { - uint64_t n; - if (!integer || !decimal(digits, UINT32_MAX, &n)) return false; - write_u32(prefix + field->offset, (uint32_t)n); - return true; - } - double value = strtod(digits, NULL); - if (!isfinite(value)) return false; - memcpy(prefix + field->offset, &value, 8); - return true; -} - -/* A JSON string's contents (after its opening quote) up to the closing one: - * its length, or 0 when it does not close or holds a control byte. */ -static size_t json_string_end(const unsigned char *p, size_t size) { - for (size_t i = 0; i < size; ++i) { - /* Eight bytes at a time while none is a quote, a backslash or a control byte. */ - const uint64_t ones = UINT64_C(0x0101010101010101), highs = UINT64_C(0x8080808080808080); - for (uint64_t w, q, s; size - i >= 8; i += 8) { - memcpy(&w, p + i, 8); - q = w ^ (ones * '"'); - s = w ^ (ones * '\\'); - if (((q - ones) & ~q & highs) | ((s - ones) & ~s & highs) | ((w - ones * 0x20) & ~w & highs)) break; - } - if (i == size) break; - if (p[i] == '"') return i + 1; - if (p[i] < 0x20) return 0; - if (p[i] == '\\' && ++i == size) return 0; - } - return 0; -} - -static int hex_digit(unsigned char c) { - return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; -} - -static bool json_code(const unsigned char *p, size_t left, uint32_t *code) { - if (left < 4) return false; - *code = 0; - for (int i = 0; i < 4; ++i) { - int d = hex_digit(p[i]); - if (d < 0) return false; - *code = *code << 4 | (uint32_t)d; - } - return true; -} - -/* Unescape a JSON string's contents into out (never longer); false for a - * bad escape or a lone surrogate. The result is checked as UTF-8 after. */ -static bool json_unescape(const unsigned char *in, size_t size, unsigned char *out, size_t *written) { - size_t o = 0; - for (size_t i = 0; i < size;) { - unsigned char b = in[i++]; - if (b != '\\') { out[o++] = b; continue; } - if (i == size) return false; - b = in[i++]; - const char *plain = strchr("\"\\/bfnrt", b); - if (b && plain) { out[o++] = (unsigned char)"\"\\/\b\f\n\r\t"[plain - "\"\\/bfnrt"]; continue; } - uint32_t code, low; - if (b != 'u' || !json_code(in + i, size - i, &code)) return false; - i += 4; - if (code >= 0xDC00 && code <= 0xDFFF) return false; - if (code >= 0xD800 && code <= 0xDBFF) { - if (size - i < 6 || in[i] != '\\' || in[i + 1] != 'u' || !json_code(in + i + 2, size - i - 2, &low) || - low < 0xDC00 || low > 0xDFFF) return false; - i += 6; - code = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); - } - if (code < 0x80) out[o++] = (unsigned char)code; - else if (code < 0x800) { out[o++] = (unsigned char)(0xC0 | code >> 6); out[o++] = (unsigned char)(0x80 | (code & 0x3F)); } - else if (code < 0x10000) { - out[o++] = (unsigned char)(0xE0 | code >> 12); - out[o++] = (unsigned char)(0x80 | (code >> 6 & 0x3F)); - out[o++] = (unsigned char)(0x80 | (code & 0x3F)); - } else { - out[o++] = (unsigned char)(0xF0 | code >> 18); - out[o++] = (unsigned char)(0x80 | (code >> 12 & 0x3F)); - out[o++] = (unsigned char)(0x80 | (code >> 6 & 0x3F)); - out[o++] = (unsigned char)(0x80 | (code & 0x3F)); - } - } - *written = o; - return true; -} - -size_t pure_json_space(const unsigned char *p, size_t at, size_t size) { - while (at < size && (p[at] == ' ' || p[at] == '\t' || p[at] == '\n' || p[at] == '\r')) ++at; - return at; -} - -/* Skip any JSON value at json[*at] (at most 64 deep): a member of an - * answer that names no field. */ -static bool skip_value(const unsigned char *json, size_t size, size_t *at, unsigned depth) { - if (*at == size || depth > 64) return false; - unsigned char b = json[*at]; - bool integer; - if (b == '"') { - size_t end = json_string_end(json + *at + 1, size - *at - 1); - if (!end) return false; - *at += 1 + end; - return true; - } - if (b == '{' || b == '[') { - unsigned char close = b == '{' ? '}' : ']'; - *at = pure_json_space(json, *at + 1, size); - for (bool first = true; *at < size && json[*at] != close; first = false) { - if (!first) { - if (json[(*at)++] != ',') return false; - *at = pure_json_space(json, *at, size); - } - if (close == '}') { - size_t key_end = *at < size && json[*at] == '"' ? json_string_end(json + *at + 1, size - *at - 1) : 0; - if (!key_end) return false; - *at = pure_json_space(json, *at + 1 + key_end, size); - if (*at == size || json[(*at)++] != ':') return false; - *at = pure_json_space(json, *at, size); - } - if (!skip_value(json, size, at, depth + 1)) return false; - *at = pure_json_space(json, *at, size); - } - if (*at == size) return false; - ++*at; - return true; - } - static const char *const words[] = {"true", "false", "null"}; - for (size_t w = 0; w < 3; ++w) - if (size - *at >= strlen(words[w]) && !memcmp(json + *at, words[w], strlen(words[w]))) { - *at += strlen(words[w]); - return true; - } - size_t length = json_number(json + *at, size - *at, &integer); - *at += length; - return length > 0; -} - -/* The field named path.key (key alone at the top), or the field count when - * none is; group says whether fields are named inside it instead. */ -static uint32_t field_named(const PureJsonMessage *b, const char *path, size_t path_size, const unsigned char *key, - size_t key_size, bool *group) { - const PureModule *m = b->module; - size_t size = path_size + !!path_size + key_size; - *group = false; - for (uint32_t f = 0; f < b->shape->field_count; ++f) { - const char *name = m->field[b->shape->field_start + f].name; - size_t length = strlen(name); - if (length < size || memcmp(name, path, path_size) || (path_size && name[path_size] != '.') || - memcmp(name + path_size + !!path_size, key, key_size)) continue; - if (length == size) return f; - if (name[size] == '.') *group = true; - } - return b->shape->field_count; -} - -/* An object's members from json[*at] (its opening brace), in the group named - * path: a field's value, or a group's object. Strings stay where they are - * (an escaped one is unescaped over itself, which never makes it longer). - * NULL, or the error to answer. */ -static const char *members(PureJsonMessage *b, unsigned char *json, size_t size, size_t *at, const char *path, size_t path_size, - unsigned depth) { - const PureModule *m = b->module; - if (*at == size || json[(*at)++] != '{') return "json"; - *at = pure_json_space(json, *at, size); - for (bool first = true; *at < size && json[*at] != '}'; first = false) { - if (!first) { - if (json[(*at)++] != ',') return "json"; - *at = pure_json_space(json, *at, size); - } - size_t key_end = *at < size && json[*at] == '"' ? json_string_end(json + *at + 1, size - *at - 1) : 0; - if (!key_end) return "json"; - const unsigned char *key = json + *at + 1; - size_t key_size = key_end - 1; - *at = pure_json_space(json, *at + 1 + key_end, size); - if (*at == size || json[(*at)++] != ':') return "json"; - *at = pure_json_space(json, *at, size); - bool group; - uint32_t f = field_named(b, path, path_size, key, key_size, &group); - if (f == b->shape->field_count && !group && b->lenient) { /* not the app's: skipped */ - if (!skip_value(json, size, at, depth)) return "json"; - *at = pure_json_space(json, *at, size); - continue; - } - if (*at < size && json[*at] == '{') { - if (!group) return f < b->shape->field_count ? "json_type" : "json_field"; - char inner[PURE_MODULE_NAME]; - size_t inner_size = path_size + !!path_size + key_size; - memcpy(inner, path, path_size); - if (path_size) inner[path_size] = '.'; - memcpy(inner + path_size + !!path_size, key, key_size); - const char *why = members(b, json, size, at, inner, inner_size, depth + 1); - if (why) return why; - *at = pure_json_space(json, *at, size); - continue; - } - if (f == b->shape->field_count) return group ? "json_type" : "json_field"; - if (b->seen[f] || (b->given && b->given[f])) return "json_field"; - b->seen[f] = true; - const PureField *field = &m->field[b->shape->field_start + f]; - bool variable = field->type == PURE_FIELD_BYTES || field->type == PURE_FIELD_TEXT, integer; - if (*at < size && json[*at] == '"') { - size_t end = json_string_end(json + *at + 1, size - *at - 1), written; - if (!end) return "json"; - if (!variable) return "json_type"; - unsigned char *value = json + *at + 1; - if (!memchr(value, '\\', end - 1)) written = end - 1; - else if (!json_unescape(value, end - 1, value, &written)) return "json"; - if (field->type == PURE_FIELD_TEXT && !pure_utf8_valid((PureSpan){value, written})) return "json"; - write_u32(b->table + 8 * f, (uint32_t)(value - b->payload)); - write_u32(b->table + 8 * f + 4, (uint32_t)written); - *at = pure_json_space(json, *at + 1 + end, size); - continue; - } - size_t length = json_number(json + *at, size - *at, &integer); - if (!length) { - static const char *const words[] = {"true", "false", "null"}; - for (size_t w = 0; w < 3 && !length; ++w) - if (size - *at >= strlen(words[w]) && !memcmp(json + *at, words[w], strlen(words[w]))) return "json_type"; - return *at < size && json[*at] == '[' ? "json_type" : "json"; - } - if (variable || !pure_json_fixed(field, json + *at, length, b->payload)) return "json_type"; - *at = pure_json_space(json, *at + length, size); - } - if (*at == size) return "json"; - ++*at; - return NULL; -} - -const char *pure_json_message(PureJsonMessage *message, unsigned char *json, size_t size, size_t *at) { - return members(message, json, size, at, "", 0, 0); -} diff --git a/prototypes/connections/message-json.h b/prototypes/connections/message-json.h deleted file mode 100644 index 61211d6..0000000 --- a/prototypes/connections/message-json.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef PURE_MESSAGE_JSON_H -#define PURE_MESSAGE_JSON_H -#include "module.h" - -/* A message's fields from a JSON object read in place (docs/research/082-json-in-place.md): - * a u32 or a number for a fixed field (put into the prefix), a string for - * bytes or text (left where it is: an escaped one is unescaped over itself, - * which never makes it longer), a group's fields as an object inside. */ -typedef struct { - const PureModule *module; - const PureInputShape *shape; - unsigned char *payload; /* the prefix; a string's start is counted from here */ - unsigned char *table; /* for each field, its start and size (u32le each) */ - const bool *given; /* fields given elsewhere (a request's query), or NULL */ - bool lenient; /* a member naming no field is skipped, value and all (an API's answer) */ - bool seen[PURE_MODULE_FIELDS]; -} PureJsonMessage; - -/* The object at json[*at] (its opening brace), *at moved past it: NULL, or - * what is wrong ("json", "json_field": unknown or given twice, "json_type"). */ -const char *pure_json_message(PureJsonMessage *, unsigned char *json, size_t size, size_t *at); -size_t pure_json_space(const unsigned char *json, size_t at, size_t size); -/* A fixed field's value from its text (a query value or a JSON number) into - * the prefix: a u32 in range, or a finite number. */ -bool pure_json_fixed(const PureField *field, const unsigned char *text, size_t size, unsigned char *prefix); -#endif diff --git a/prototypes/connections/module-list.c b/prototypes/connections/module-list.c deleted file mode 100644 index 2cd2b1c..0000000 --- a/prototypes/connections/module-list.c +++ /dev/null @@ -1,428 +0,0 @@ -#include "module-list.h" -#include "module-text.h" -#include -#include -#include - -struct PureList { - unsigned fields; - uint8_t text_fields[16]; - PureCell **items; - size_t count, capacity; - PureDraft *draft; /* the stored copy, or NULL */ - size_t header; /* draft bytes before the first item */ - uint64_t id, version; /* which list, and a number every change moves on */ - uint64_t shares; /* copies made so far: an item made before the last one is shared */ - PureRetired *retired; /* where shared items go when the list lets them go */ - unsigned refs; /* a shared copy's references */ - bool copy; /* a shared copy: it owns its item array, not its items */ -}; - -typedef enum { INSERTED, REMOVED, CHANGED, REPLACED } Change; -typedef struct { - Change change; - PureList *list; - size_t at; - unsigned field; - PureCell *item; /* REMOVED: the removed item; REPLACED: the shared item a copy replaced */ - PureCell old; /* CHANGED: the replaced cell */ -} Entry; -typedef struct { PureCell *item; uint16_t text; uint8_t fields; } Retired; - -static uint64_t lists_made; - -/* An item's cells follow the list's share count when the item was made. One - * made before the last share may be read by a shared copy, so it never - * changes again (a change copies it) and is retired instead of freed. */ -static PureCell *new_item(const PureList *list) { - uint64_t *made = calloc(1, sizeof(*made) + (list->fields ? list->fields : 1) * sizeof(PureCell)); - if (!made) return NULL; - *made = list->shares; - return (PureCell *)(made + 1); -} - -static bool shared(const PureList *list, const PureCell *item) { return ((const uint64_t *)item)[-1] < list->shares; } - -static void free_cells(PureCell *item, unsigned fields, unsigned text) { - for (unsigned f = 0; f < fields; ++f) - if (text >> f & 1) free(item[f].text); - free((uint64_t *)item - 1); -} - -static unsigned text_mask(const PureList *list) { - unsigned text = 0; - for (unsigned f = 0; f < list->fields; ++f) text |= (unsigned)(list->text_fields[f] != 0) << f; - return text; -} - -static void free_item(const PureList *list, PureCell *item) { - if (item) free_cells(item, list->fields, text_mask(list)); -} - -/* The list no longer has item: free it, or retire it while a copy may read it. */ -static void let_go(PureList *list, PureCell *item) { - if (!shared(list, item)) { free_item(list, item); return; } - PureRetired *retired = list->retired; - if (retired->count == retired->capacity) { - size_t capacity = retired->capacity ? 2 * retired->capacity : 16; - Retired *grown = realloc(retired->entries, capacity * sizeof(*grown)); - if (!grown) return; /* kept for good rather than freed while read */ - retired->entries = grown; - retired->capacity = capacity; - } - ((Retired *)retired->entries)[retired->count++] = (Retired){item, (uint16_t)text_mask(list), (uint8_t)list->fields}; -} - -void pure_retired_free(PureRetired *retired) { - Retired *entries = retired->entries; - for (size_t i = 0; i < retired->count; ++i) free_cells(entries[i].item, entries[i].fields, entries[i].text); - free(entries); - *retired = (PureRetired){0}; -} - -PureList *pure_list_new(unsigned fields, const uint8_t *text_fields) { - if (fields > 16) return NULL; - PureList *list = calloc(1, sizeof(*list)); - if (!list) return NULL; - list->fields = fields; - memcpy(list->text_fields, text_fields, fields); - list->id = __atomic_add_fetch(&lists_made, 1, __ATOMIC_RELAXED); - return list; -} - -void pure_list_free(PureList *list) { - if (!list) return; - if (list->copy) { - if (__atomic_sub_fetch(&list->refs, 1, __ATOMIC_ACQ_REL)) return; - } else - for (size_t i = 0; i < list->count; ++i) let_go(list, list->items[i]); - free(list->items); - free(list); -} - -size_t pure_list_count(const PureList *list) { return list->count; } - -PureList *pure_list_share(PureList *list, PureRetired *retired) { - PureList *copy = malloc(sizeof(*copy)); - PureCell **items = list->count ? malloc(list->count * sizeof(*items)) : NULL; - if (!copy || (list->count && !items)) { - free(copy); - free(items); - return NULL; - } - if (list->count) memcpy(items, list->items, list->count * sizeof(*items)); - *copy = (PureList){.fields = list->fields, .items = items, .count = list->count, .capacity = list->count, - .id = list->id, .version = list->version, .refs = 1, .copy = true}; - memcpy(copy->text_fields, list->text_fields, sizeof(copy->text_fields)); - list->retired = retired; - ++list->shares; - return copy; -} - -void pure_list_hold(PureList *copy) { __atomic_add_fetch(©->refs, 1, __ATOMIC_RELAXED); } - -bool pure_list_current(const PureList *copy, const PureList *list) { - return copy->id == list->id && copy->version == list->version; -} - -static bool log_change(PureUndo *undo, Entry entry) { - if (undo->count == undo->capacity) { - size_t capacity = undo->capacity ? 2 * undo->capacity : 16; - Entry *entries = realloc(undo->entries, capacity * sizeof(*entries)); - if (!entries) return false; - undo->entries = entries; - undo->capacity = capacity; - } - ((Entry *)undo->entries)[undo->count++] = entry; - return true; -} - -static bool copy_cell(const PureList *list, unsigned field, PureCell from, PureCell *to) { - *to = (PureCell){.num = list->text_fields[field] ? 0 : from.num}; - if (!list->text_fields[field] || !from.size) return true; - to->text = malloc(from.size); - if (!to->text) return false; - memcpy(to->text, from.text, from.size); - to->size = from.size; - return true; -} - -/* ---- The stored form: each field in order, a number as its binary64 bits - * (little-endian), a text as a u32le length and its bytes. */ - -static size_t cell_bytes(const PureList *list, unsigned field, PureCell cell) { - return list->text_fields[field] ? 4 + cell.size : 8; -} - -static size_t item_bytes(const PureList *list, const PureCell *item) { - size_t size = 0; - for (unsigned f = 0; f < list->fields; ++f) size += cell_bytes(list, f, item[f]); - return size; -} - -/* Where an item starts in the draft. Linear in the items before it. */ -static size_t item_offset(const PureList *list, size_t at) { - size_t offset = list->header; - for (size_t i = 0; i < at; ++i) offset += item_bytes(list, list->items[i]); - return offset; -} - -static unsigned char *encode_cell(const PureList *list, unsigned field, PureCell cell, unsigned char *out) { - uint64_t bits = cell.size; - unsigned width = 4; - if (!list->text_fields[field]) { memcpy(&bits, &cell.num, sizeof(bits)); width = 8; } - for (unsigned i = 0; i < width; ++i) *out++ = (unsigned char)(bits >> (8 * i)); - if (list->text_fields[field] && cell.size) { memcpy(out, cell.text, cell.size); out += cell.size; } - return out; -} - -/* Replace remove bytes of the draft, within bytes into the item at `at` (or - * after the last item), with bytes; nothing to do without a draft. */ -static PureResult mirror(PureList *list, size_t at, size_t within, size_t remove, const unsigned char *bytes, - size_t size) { - if (!list->draft) return PURE_OK; - return pure_draft_splice(list->draft, item_offset(list, at) + within, remove, (PureSpan){bytes, size}); -} - -void pure_list_mirror(PureList *list, PureDraft *draft, size_t header) { - list->draft = draft; - list->header = header; -} - -PURE_COLD PureResult pure_list_write(PureList *list) { - size_t size = 0; - for (size_t i = 0; i < list->count; ++i) size += item_bytes(list, list->items[i]); - unsigned char *bytes = size ? malloc(size) : NULL, *at = bytes; - if (size && !bytes) return PURE_ALLOCATION; - for (size_t i = 0; i < list->count; ++i) - for (unsigned f = 0; f < list->fields; ++f) at = encode_cell(list, f, list->items[i][f], at); - PureResult result = mirror(list, 0, 0, 0, bytes, size); - free(bytes); - return result; -} - -PURE_COLD PureResult pure_list_read(PureList *list, const unsigned char *bytes, size_t size) { - PureCell cells[16]; - for (size_t at = 0; at < size;) { - for (unsigned f = 0; f < list->fields; ++f) { - size_t width = list->text_fields[f] ? 4 : 8; - if (size - at < width) return PURE_STORE_ERROR; - uint64_t bits = 0; - for (size_t i = 0; i < width; ++i) bits |= (uint64_t)bytes[at + i] << (8 * i); - at += width; - if (!list->text_fields[f]) { - cells[f] = (PureCell){0}; - memcpy(&cells[f].num, &bits, sizeof(bits)); - if (!isfinite(cells[f].num)) return PURE_STORE_ERROR; - continue; - } - if (bits > size - at || !pure_utf8_valid((PureSpan){bytes + at, (size_t)bits})) - return PURE_STORE_ERROR; - cells[f] = (PureCell){0, (unsigned char *)bytes + at, (size_t)bits}; - at += (size_t)bits; - } - PureResult result = pure_list_insert(list, list->count, cells, NULL); - if (result != PURE_OK) return result; - } - return PURE_OK; -} - -static void place(PureList *list, size_t at, PureCell *item) { - memmove(list->items + at + 1, list->items + at, (list->count - at) * sizeof(*list->items)); - list->items[at] = item; - ++list->count; - ++list->version; -} - -static PureCell *take(PureList *list, size_t at) { - PureCell *item = list->items[at]; - memmove(list->items + at, list->items + at + 1, (list->count - at - 1) * sizeof(*list->items)); - --list->count; - ++list->version; - return item; -} - -PureResult pure_list_insert(PureList *list, size_t at, const PureCell *cells, PureUndo *undo) { - if (at > list->count || list->count == PURE_LIST_ITEMS) return PURE_LIMIT; - if (list->count == list->capacity) { - size_t capacity = list->capacity ? 2 * list->capacity : 8; - PureCell **items = realloc(list->items, capacity * sizeof(*items)); - if (!items) return PURE_ALLOCATION; - list->items = items; - list->capacity = capacity; - } - PureCell *item = new_item(list); - if (!item) return PURE_ALLOCATION; - for (unsigned f = 0; f < list->fields; ++f) - if (!copy_cell(list, f, cells[f], &item[f])) { free_item(list, item); return PURE_ALLOCATION; } - size_t size = list->draft ? item_bytes(list, item) : 0; - unsigned char *bytes = size ? malloc(size) : NULL, *end = bytes; - if (size && !bytes) { free_item(list, item); return PURE_ALLOCATION; } - for (unsigned f = 0; bytes && f < list->fields; ++f) end = encode_cell(list, f, item[f], end); - if (undo && !log_change(undo, (Entry){.change = INSERTED, .list = list, .at = at})) { - free(bytes); - free_item(list, item); - return PURE_ALLOCATION; - } - PureResult result = mirror(list, at, 0, 0, bytes, size); - free(bytes); - if (result != PURE_OK) { - if (undo) --undo->count; - free_item(list, item); - return result; - } - place(list, at, item); - return PURE_OK; -} - -PureResult pure_list_remove(PureList *list, size_t at, PureUndo *undo) { - if (at >= list->count) return PURE_NOT_FOUND; - if (!log_change(undo, (Entry){.change = REMOVED, .list = list, .at = at, .item = list->items[at]})) - return PURE_ALLOCATION; - PureResult result = mirror(list, at, 0, item_bytes(list, list->items[at]), NULL, 0); - if (result != PURE_OK) { --undo->count; return result; } - take(list, at); - return PURE_OK; -} - -/* A shared item's copy with every field but one (left empty). */ -static PureCell *copy_item(const PureList *list, const PureCell *item, unsigned except) { - PureCell *fresh = new_item(list); - for (unsigned f = 0; fresh && f < list->fields; ++f) - if (f != except && !copy_cell(list, f, item[f], &fresh[f])) { - free_item(list, fresh); - fresh = NULL; - } - return fresh; -} - -PureResult pure_list_set(PureList *list, size_t at, unsigned field, PureCell value, PureUndo *undo) { - if (at >= list->count) return PURE_NOT_FOUND; - PureCell copy, *item = list->items[at]; - if (!copy_cell(list, field, value, ©)) return PURE_ALLOCATION; - PureCell *fresh = shared(list, item) ? copy_item(list, item, field) : item; - size_t size = list->draft ? cell_bytes(list, field, copy) : 0; - unsigned char *bytes = size ? malloc(size) : NULL; - Entry entry = fresh == item ? (Entry){.change = CHANGED, .list = list, .at = at, .field = field, .old = item[field]} : - (Entry){.change = REPLACED, .list = list, .at = at, .item = item}; - if (!fresh || (size && !bytes) || !log_change(undo, entry)) { - free(bytes); - free(copy.text); - if (fresh != item) free_item(list, fresh); - return PURE_ALLOCATION; - } - if (bytes) encode_cell(list, field, copy, bytes); - size_t within = 0; - for (unsigned f = 0; f < field; ++f) within += cell_bytes(list, f, item[f]); - PureResult result = mirror(list, at, within, cell_bytes(list, field, item[field]), bytes, size); - free(bytes); - if (result != PURE_OK) { - --undo->count; - free(copy.text); - if (fresh != item) free_item(list, fresh); - return result; - } - fresh[field] = copy; - list->items[at] = fresh; - ++list->version; - return PURE_OK; -} - -PureCell pure_list_get(const PureList *list, size_t at, unsigned field) { - return list->items[at][field]; -} - -/* How two cells of a field order: numbers by value (-0 and +0 are one key), - * texts by their bytes. */ -static int order(const PureList *list, unsigned field, PureCell a, PureCell b) { - if (!list->text_fields[field]) return (a.num > b.num) - (a.num < b.num); - size_t shorter = a.size < b.size ? a.size : b.size; - int bytes = shorter ? memcmp(a.text, b.text, shorter) : 0; - return bytes ? (bytes > 0) - (bytes < 0) : (a.size > b.size) - (a.size < b.size); -} - -size_t pure_list_seek(const PureList *list, unsigned field, PureCell key, bool *found) { - size_t low = 0, high = list->count; - while (low < high) { - size_t middle = low + (high - low) / 2; - if (order(list, field, list->items[middle][field], key) < 0) low = middle + 1; - else high = middle; - } - *found = low < list->count && !order(list, field, list->items[low][field], key); - return low; -} - -/* Positions sorted by their items' field, stably (spare holds count). */ -PURE_COLD static void merge_sort(const PureList *list, unsigned field, size_t *at, size_t *spare, size_t count) { - if (count < 2) return; - size_t half = count / 2, i = 0, j = half, k = 0; - merge_sort(list, field, at, spare, half); - merge_sort(list, field, at + half, spare, count - half); - while (i < half && j < count) - spare[k++] = order(list, field, list->items[at[j]][field], list->items[at[i]][field]) < 0 ? at[j++] : at[i++]; - while (i < half) spare[k++] = at[i++]; - while (j < count) spare[k++] = at[j++]; - memcpy(at, spare, count * sizeof(*at)); -} - -PURE_COLD PureResult pure_list_sorted(const PureList *list, unsigned field, PureList **out, size_t *moved) { - *out = NULL; - bool ordered = true; - for (size_t i = 1; i < list->count && ordered; ++i) - ordered = order(list, field, list->items[i - 1][field], list->items[i][field]) < 0; - if (ordered) return PURE_OK; - size_t *at = malloc(2 * list->count * sizeof(*at)); - if (!at) return PURE_ALLOCATION; - for (size_t i = 0; i < list->count; ++i) at[i] = i; - merge_sort(list, field, at, at + list->count, list->count); - PureResult result = PURE_OK; - for (size_t i = 1; i < list->count && result == PURE_OK; ++i) - if (!order(list, field, list->items[at[i - 1]][field], list->items[at[i]][field])) result = PURE_CONFLICT; - PureList *sorted = result == PURE_OK ? pure_list_new(list->fields, list->text_fields) : NULL; - if (result == PURE_OK && !sorted) result = PURE_ALLOCATION; - for (size_t i = 0; result == PURE_OK && i < list->count; ++i) { - result = pure_list_insert(sorted, i, list->items[at[i]], NULL); - moved[at[i]] = i; - } - free(at); - if (result != PURE_OK) { - pure_list_free(sorted); - return result; - } - *out = sorted; - return PURE_OK; -} - -void pure_undo_rollback(PureUndo *undo, size_t mark) { - Entry *entries = undo->entries; - for (size_t i = undo->count; i-- > mark;) { - Entry *e = &entries[i]; - if (e->change == INSERTED) free_item(e->list, take(e->list, e->at)); - else if (e->change == REMOVED) place(e->list, e->at, e->item); /* capacity is still there */ - else if (e->change == REPLACED) { - free_item(e->list, e->list->items[e->at]); - e->list->items[e->at] = e->item; - ++e->list->version; - } else { - if (e->list->text_fields[e->field]) free(e->list->items[e->at][e->field].text); - e->list->items[e->at][e->field] = e->old; - ++e->list->version; - } - } - undo->count = mark; -} - -void pure_undo_commit(PureUndo *undo) { - Entry *entries = undo->entries; - for (size_t i = 0; i < undo->count; ++i) { - Entry *e = &entries[i]; - if (e->change == REMOVED || e->change == REPLACED) let_go(e->list, e->item); - else if (e->change == CHANGED && e->list->text_fields[e->field]) free(e->old.text); - } - undo->count = 0; -} - -void pure_undo_free(PureUndo *undo) { - free(undo->entries); - *undo = (PureUndo){0}; -} diff --git a/prototypes/connections/module-list.h b/prototypes/connections/module-list.h deleted file mode 100644 index a721ff0..0000000 --- a/prototypes/connections/module-list.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef PURE_MODULE_LIST_H -#define PURE_MODULE_LIST_H -#include "result.h" -#include "state.h" -#include -#include -#include - -/* Executor paths that run once per module or app (loading, verifying, - * starting, replacing, reading stored states) are built for size; events - * stay built for speed. */ -#if defined(__GNUC__) || defined(__clang__) -#define PURE_COLD __attribute__((cold)) -#else -#define PURE_COLD -#endif - -/* Runtime-owned lists of records for the module executor. An item is one - * cell per field: a number, or text bytes the list owns. Every change made - * during an event is logged; a failed event undoes the log in reverse and a - * successful one frees what it replaced. */ -typedef struct { double num; unsigned char *text; size_t size; } PureCell; -typedef struct PureList PureList; -typedef struct { void *entries; size_t count, capacity; } PureUndo; -/* Items a list let go while a shared copy may still read them. */ -typedef struct { void *entries; size_t count, capacity; } PureRetired; - -enum { PURE_LIST_ITEMS = 65536 }; - -PureList *pure_list_new(unsigned fields, const uint8_t *text_fields); -void pure_list_free(PureList *); -size_t pure_list_count(const PureList *); -/* Share the items as they are with a read-only copy (for readers on other - * threads), without copying them: from then on the list copies an item it - * changes, and items it lets go go to retired instead of being freed, since - * the copy may still read them. The copy is freed with pure_list_free, on any - * thread, when its last reference goes (pure_list_hold takes one more). Only - * between events (no change waits to be committed or undone). */ -PureList *pure_list_share(PureList *, PureRetired *retired); -void pure_list_hold(PureList *copy); -/* Whether copy shares list's items as they are now. */ -bool pure_list_current(const PureList *copy, const PureList *list); -/* Free items retired while no copy that shared them remains. */ -void pure_retired_free(PureRetired *); -/* Insert a copy of cells (text bytes are copied) before position at. A NULL - * log means the change is not undone. */ -PureResult pure_list_insert(PureList *, size_t at, const PureCell *cells, PureUndo *); -PureResult pure_list_remove(PureList *, size_t at, PureUndo *); -PureResult pure_list_set(PureList *, size_t at, unsigned field, PureCell value, PureUndo *); -PureCell pure_list_get(const PureList *, size_t at, unsigned field); -/* In a list ordered by field (a keyed list): the first position whose item's - * field does not order below key (numbers by value, texts by bytes), and - * whether that item's field is key. */ -size_t pure_list_seek(const PureList *, unsigned field, PureCell key, bool *found); -/* The list's items ordered by field, as a keyed list keeps them: *out is NULL - * when they already are, or a new list in that order with moved[position - * before] = position after for every item. CONFLICT when two items have the - * same key (nothing is made). */ -PureResult pure_list_sorted(const PureList *, unsigned field, PureList **out, size_t *moved); -/* A list may keep a stored copy in a draft: header bytes, then each item's - * fields in order (a number as its binary64 bits, little-endian; a text as a - * u32le length and its bytes). Every change splices the draft as well, and a - * change the draft refuses (such as its size limit) does not happen. Undoing - * an event changes only the list: the caller restores the draft's version. */ -void pure_list_mirror(PureList *, PureDraft *, size_t header); -/* Store every item after the header (the draft holds just the header). */ -PureResult pure_list_write(PureList *); -/* Append the items stored in bytes; each field is checked. Before mirroring. */ -PureResult pure_list_read(PureList *, const unsigned char *bytes, size_t size); -/* Undo the changes logged after mark (a count taken earlier), newest first. */ -void pure_undo_rollback(PureUndo *, size_t mark); -void pure_undo_commit(PureUndo *); -void pure_undo_free(PureUndo *); -#endif diff --git a/prototypes/connections/module-save.c b/prototypes/connections/module-save.c deleted file mode 100644 index 4149316..0000000 --- a/prototypes/connections/module-save.c +++ /dev/null @@ -1,488 +0,0 @@ -#include "module-save.h" -#include "module-text.h" -#include -#include -#include -#include - -/* Stored forms. values: "PVS1", then per state {u8 type, u8 name length, - * name, binary64 bits (little-endian)}. A list: "PLS1", u8 field count, per - * field {u8 0 number | 1 text, u8 name length, name}, then its items - * (module-list.h). A text state: its UTF-8 bytes. */ -static const unsigned char values_magic[4] = {'P', 'V', 'S', '1'}, list_magic[4] = {'P', 'L', 'S', '1'}; -/* links: "PLK1"; u8 outputs, each {u8 name length, name, u8 field count, the - * fields' types, u64le last number} (the module's outputs, then those it - * dropped, without fields); u16le sources, each {u8 length, the name, u64le - * last number heard}; then the sent messages to the end, each {u8 output (in - * the table), u64le number, u32le size, payload}. */ -static const unsigned char links_magic[4] = {'P', 'L', 'K', '1'}; -static const PureVersion unsaved = {UINT64_MAX, UINT64_MAX}; - -bool pure_save_enabled(const PureApp *app) { return app->data.state && app->data.save; } - -static PureResult open_state(PureApp *app, const char *prefix, const char *name, PureDraft **out) { - char key[64]; - snprintf(key, sizeof(key), "%s%s", prefix, name); - return app->data.state(app->data.context, key, out); -} - -/* The whole value of a draft, contiguous; free it. */ -typedef struct { unsigned char *bytes; size_t used; } Whole; - -static bool gather(void *context, PureSpan span) { - Whole *w = context; - memcpy(w->bytes + w->used, span.data, span.size); - w->used += span.size; - return true; -} - -static PureResult read_whole(const PureDraft *draft, unsigned char **out, size_t *size) { - *size = pure_draft_size(draft); - Whole w = {malloc(*size ? *size : 1), 0}; - if (!w.bytes) return PURE_ALLOCATION; - PureResult result = pure_draft_read(draft, 0, *size, gather, &w); - if (result != PURE_OK) { free(w.bytes); return result; } - *out = w.bytes; - return PURE_OK; -} - -/* One named entry of a stored form: u8 kind, u8 name length, the name (an - * identifier), then width more bytes. */ -PURE_COLD static bool named(const unsigned char *bytes, size_t size, size_t *at, size_t width, uint8_t *kind, - char name[PURE_MODULE_NAME]) { - if (size - *at < 2) return false; - size_t length = bytes[*at + 1]; - if (size - *at - 2 < length + width || !length || length >= PURE_MODULE_NAME) return false; - for (size_t i = 0; i < length; ++i) { - unsigned char c = bytes[*at + 2 + i]; - bool letter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; - if (!letter && !(i && c >= '0' && c <= '9')) return false; - } - *kind = bytes[*at]; - memcpy(name, bytes + *at + 2, length); - name[length] = 0; - *at += 2 + length; - return true; -} - -PURE_COLD PureResult pure_save_open_text(PureApp *app, const PureModule *m, uint32_t state, PureDraft **out, - PureVersion *saved) { - PureResult result = open_state(app, "text-", m->state_names[state], out); - if (result != PURE_OK) return result; - *saved = pure_draft_version(*out); - if (!pure_draft_revision(*out)) { - uint32_t constant = m->state_ref[state]; - PureSpan initial = {m->constant_data + m->constant_offset[constant], m->constant_length[constant]}; - result = pure_draft_splice(*out, 0, pure_draft_size(*out), initial); - *saved = unsaved; - } else { - unsigned char *bytes; - size_t size; - result = read_whole(*out, &bytes, &size); - if (result == PURE_OK) { - if (!pure_utf8_valid((PureSpan){bytes, size})) result = PURE_STORE_ERROR; - free(bytes); - } - } - if (result != PURE_OK) { pure_draft_close(*out); *out = NULL; } - return result; -} - -static size_t list_header(const PureModule *m, uint32_t record, unsigned char *out) { - const PureRecord *r = &m->record[record]; - size_t at = 0; - memcpy(out, list_magic, 4); - at = 4; - out[at++] = (unsigned char)r->count; - for (unsigned f = 0; f < r->count; ++f) { - const char *name = m->record_field_names[r->first + f]; - out[at++] = m->record_field_text[r->first + f]; - out[at++] = (unsigned char)strlen(name); - memcpy(out + at, name, strlen(name)); - at += strlen(name); - } - return at; -} - -PURE_COLD PureResult pure_save_rewrite_list(const PureModule *m, uint32_t state, PureList *list, PureDraft *draft) { - unsigned char header[5 + PURE_RECORD_FIELDS * (2 + PURE_MODULE_NAME)]; - size_t length = list_header(m, m->state_ref[state], header); - PureVersion before = pure_draft_version(draft); - pure_draft_hold(draft, true); - PureResult result = pure_draft_splice(draft, 0, pure_draft_size(draft), (PureSpan){header, length}); - pure_list_mirror(list, draft, length); - if (result == PURE_OK) result = pure_list_write(list); - if (result != PURE_OK) { - pure_draft_restore(draft, before); - pure_list_mirror(list, NULL, 0); - } - pure_draft_hold(draft, false); - return result; -} - -PURE_COLD PureResult pure_save_open_list(PureApp *app, const PureModule *m, uint32_t state, PureList **out, PureDraft **draft, - PureVersion *saved) { - uint32_t record = m->state_ref[state]; - const PureRecord *r = &m->record[record]; - *out = NULL; - PureResult result = open_state(app, "list-", m->state_names[state], draft); - if (result != PURE_OK) return result; - unsigned char *bytes = NULL; - size_t size = 0; - if (pure_draft_revision(*draft)) result = read_whole(*draft, &bytes, &size); - /* The stored record: its fields' kinds and names. */ - char names[PURE_RECORD_FIELDS][PURE_MODULE_NAME] = {{0}}; - uint8_t text[PURE_RECORD_FIELDS] = {0}; - unsigned count = 0; - size_t at = 0; - bool stored = result == PURE_OK && bytes; - if (stored) { - bool ok = size >= 5 && !memcmp(bytes, list_magic, 4) && bytes[4] <= PURE_RECORD_FIELDS; - count = ok ? bytes[4] : 0; - at = 5; - for (unsigned f = 0; ok && f < count; ++f) { - ok = named(bytes, size, &at, 0, &text[f], names[f]) && text[f] <= 1; - for (unsigned k = 0; k < f; ++k) ok = ok && strcmp(names[k], names[f]); - } - if (!ok) result = PURE_STORE_ERROR; - } - PureList *from = NULL; - if (result == PURE_OK && stored) { - from = pure_list_new(count, text); - result = from ? pure_list_read(from, bytes + at, size - at) : PURE_ALLOCATION; - } - free(bytes); - bool same = stored && count == r->count; - for (unsigned f = 0; same && f < count; ++f) - same = text[f] == m->record_field_text[r->first + f] && !strcmp(names[f], m->record_field_names[r->first + f]); - if (result == PURE_OK && same) { - *out = from; - pure_list_mirror(from, *draft, at); - *saved = pure_draft_version(*draft); - return PURE_OK; - } - if (result == PURE_OK) { - /* New, or stored with another record: its items move by field name. */ - const char *field_names[PURE_RECORD_FIELDS]; - for (unsigned f = 0; f < count; ++f) field_names[f] = names[f]; - if (from) result = pure_list_migrate(out, from, field_names, text, count, m, record); - else if (!(*out = pure_list_new(r->count, m->record_field_text + r->first))) result = PURE_ALLOCATION; - } - pure_list_free(from); - if (result == PURE_OK) result = pure_save_rewrite_list(m, state, *out, *draft); - if (result != PURE_OK) { - pure_list_free(*out); - pure_draft_close(*draft); - *out = NULL; - *draft = NULL; - return result; - } - *saved = unsaved; - return PURE_OK; -} - -static bool scalar(uint8_t type) { - return type == PURE_STATE_NUM || type == PURE_STATE_CURSOR || type == PURE_STATE_INDEX; -} - -/* A stored value still fits its state: a cursor on a character boundary of - * its text, an index on an item of its list (or 0), a finite number. */ -PURE_COLD static bool fits(const PureModule *m, PureDraft *const *texts, PureList *const *lists, uint32_t state, double value) { - if (!isfinite(value)) return false; - if (m->state_type[state] == PURE_STATE_NUM) return true; - if (value < 0 || value != floor(value)) return false; - if (m->state_type[state] == PURE_STATE_INDEX) { - size_t count = pure_list_count(lists[m->state_ref[state]]); - return value <= (count ? (double)(count - 1) : 0); - } - const PureDraft *text = texts[m->state_ref[state]]; - size_t size = pure_draft_size(text), at = (size_t)value; - if (value > (double)size) return false; - if (at == size) return true; - unsigned char byte = 0; - Whole w = {&byte, 0}; - return pure_draft_read(text, at, 1, gather, &w) == PURE_OK && (byte & 0xC0) != 0x80; -} - -static size_t encode_values(const PureModule *m, const double *values, unsigned char *out) { - size_t at = 4; - memcpy(out, values_magic, 4); - for (uint32_t i = 0; i < m->states; ++i) { - if (!scalar(m->state_type[i])) continue; - size_t length = strlen(m->state_names[i]); - uint64_t bits; - memcpy(&bits, &values[i], sizeof(bits)); - out[at++] = m->state_type[i]; - out[at++] = (unsigned char)length; - memcpy(out + at, m->state_names[i], length); - at += length; - for (unsigned b = 0; b < 8; ++b) out[at++] = (unsigned char)(bits >> (8 * b)); - } - return at; -} - -enum { VALUES_BYTES = 4 + PURE_MODULE_STATES * (2 + PURE_MODULE_NAME + 8) }; - -PURE_COLD PureResult pure_save_open_values(PureApp *app, const PureModule *m, double *values, const int *kept, - PureDraft *const *texts, PureList *const *lists) { - bool scalars = false, fresh = false; - for (uint32_t i = 0; i < m->states; ++i) - if (scalar(m->state_type[i])) { - scalars = true; - fresh = fresh || kept[i] < 0; - } - if (!scalars) return PURE_OK; - PureResult result = app->stored_values ? PURE_OK : open_state(app, "", "values", &app->stored_values); - if (result != PURE_OK || !fresh || !pure_draft_revision(app->stored_values)) return result; - unsigned char *bytes; - size_t size; - if ((result = read_whole(app->stored_values, &bytes, &size)) != PURE_OK) return result; - bool ok = size >= 4 && !memcmp(bytes, values_magic, 4); - for (size_t at = 4; ok && at < size;) { - char name[PURE_MODULE_NAME]; - uint8_t type; - if (!(ok = named(bytes, size, &at, 8, &type, name))) break; - uint64_t bits = 0; - for (unsigned b = 0; b < 8; ++b) bits |= (uint64_t)bytes[at + b] << (8 * b); - at += 8; - double value; - memcpy(&value, &bits, sizeof(value)); - int state = pure_module_state(m, name); - if (state >= 0 && kept[state] < 0 && m->state_type[state] == type && scalar(type) && - fits(m, texts, lists, (uint32_t)state, value)) - values[state] = value; - } - free(bytes); - return ok ? PURE_OK : PURE_STORE_ERROR; -} - -static bool same_version(PureVersion a, PureVersion b) { return a.ref == b.ref && a.bytes == b.bytes; } - -static unsigned char *put_le(unsigned char *at, uint64_t value, unsigned width) { - for (unsigned i = 0; i < width; ++i) *at++ = (unsigned char)(value >> (8 * i)); - return at; -} - -static uint64_t get_le(const unsigned char *at, unsigned width) { - uint64_t value = 0; - for (unsigned i = 0; i < width; ++i) value |= (uint64_t)at[i] << (8 * i); - return value; -} - -/* The links as stored; NULL when memory runs out. */ -static unsigned char *encode_links(const PureApp *app, size_t *size) { - const PureModule *m = app->module; - unsigned outputs = 0; - uint8_t table[PURE_MODULE_INPUTS]; - size_t total = 4 + 1 + 2; - for (uint32_t i = 0; i < m->inputs; ++i) - if (m->input[i].kind == PURE_KIND_OUTPUT) { - table[i] = (uint8_t)outputs++; - total += 2 + strlen(m->input_names[i]) + m->input[i].field_count + 8; - } - for (unsigned c = 0; c < app->carried_count; ++c) total += 2 + strlen(app->carried[c].name) + 8; - for (unsigned h = 0; h < app->heard_count; ++h) total += 1 + strlen(app->heard[h].source) + 8; - for (size_t s = 0; s < app->sent_count; ++s) - if (!app->sent[s].trimmed) total += 1 + 8 + 4 + app->sent[s].size; - unsigned char *bytes = malloc(total), *at = bytes; - if (!bytes) return NULL; - memcpy(at, links_magic, 4); - at += 4; - *at++ = (unsigned char)(outputs + app->carried_count); - for (uint32_t i = 0; i < m->inputs; ++i) { - if (m->input[i].kind != PURE_KIND_OUTPUT) continue; - size_t length = strlen(m->input_names[i]); - *at++ = (unsigned char)length; - memcpy(at, m->input_names[i], length); - at += length; - *at++ = (unsigned char)m->input[i].field_count; - for (uint32_t f = 0; f < m->input[i].field_count; ++f) *at++ = m->field[m->input[i].field_start + f].type; - at = put_le(at, app->last_sent[i], 8); - } - for (unsigned c = 0; c < app->carried_count; ++c) { - size_t length = strlen(app->carried[c].name); - *at++ = (unsigned char)length; - memcpy(at, app->carried[c].name, length); - at += length; - *at++ = 0; - at = put_le(at, app->carried[c].last, 8); - } - at = put_le(at, app->heard_count, 2); - for (unsigned h = 0; h < app->heard_count; ++h) { - size_t length = strlen(app->heard[h].source); - *at++ = (unsigned char)length; - memcpy(at, app->heard[h].source, length); - at = put_le(at + length, app->heard[h].seq, 8); - } - for (size_t s = 0; s < app->sent_count; ++s) { - const struct PureSent *sent = &app->sent[s]; - if (sent->trimmed) continue; - *at++ = table[sent->output]; - at = put_le(at, sent->seq, 8); - at = put_le(at, sent->size, 4); - if (sent->size) memcpy(at, sent->payload, sent->size); - at += sent->size; - } - *size = (size_t)(at - bytes); - return bytes; -} - -/* The stored links of a starting app: outputs by name (an output whose - * fields changed or that the module lacks refuses the start while it has - * sends left: CONFLICT; one without sends keeps its number for later), - * sources and sent messages. */ -PURE_COLD PureResult pure_save_open_links(PureApp *app, const PureModule *m) { - PureResult result = app->stored_links ? PURE_OK : open_state(app, "", "links", &app->stored_links); - if (result != PURE_OK || !pure_draft_revision(app->stored_links)) return result; - unsigned char *bytes; - size_t size; - if ((result = read_whole(app->stored_links, &bytes, &size)) != PURE_OK) return result; - int output[256]; - bool fits[256]; - size_t at = 5; - unsigned outputs = size >= 5 && !memcmp(bytes, links_magic, 4) ? bytes[4] : 0; - result = size >= 5 && !memcmp(bytes, links_magic, 4) ? PURE_OK : PURE_STORE_ERROR; - for (unsigned t = 0; t < outputs && result == PURE_OK; ++t) { - char name[PURE_MODULE_NAME]; - size_t length = at < size ? bytes[at] : 0; - if (!length || length >= PURE_MODULE_NAME || size - at < 2 + length) { result = PURE_STORE_ERROR; break; } - memcpy(name, bytes + at + 1, length); - name[length] = 0; - unsigned fields = bytes[at + 1 + length]; - at += 2 + length; - if (size - at < fields + 8) { result = PURE_STORE_ERROR; break; } - int i = pure_module_input(m, name); - output[t] = i >= 0 && m->input[i].kind == PURE_KIND_OUTPUT ? i : -1; - fits[t] = output[t] >= 0 && m->input[i].field_count == fields; - for (unsigned f = 0; fits[t] && f < fields; ++f) fits[t] = m->field[m->input[i].field_start + f].type == bytes[at + f]; - uint64_t last = get_le(bytes + at + fields, 8); - at += fields + 8; - if (output[t] >= 0) app->last_sent[output[t]] = last; - else if (last) { - struct PureCarried *grown = realloc(app->carried, (app->carried_count + 1) * sizeof(*grown)); - if (!grown) { result = PURE_ALLOCATION; break; } - app->carried = grown; - memcpy(grown[app->carried_count].name, name, length + 1); - grown[app->carried_count++].last = last; - } - } - unsigned sources = result == PURE_OK && size - at >= 2 ? (unsigned)get_le(bytes + at, 2) : 0; - if (result == PURE_OK && size - at < 2) result = PURE_STORE_ERROR; - at += 2; - if (result == PURE_OK && sources && !(app->heard = calloc(sources, sizeof(*app->heard)))) result = PURE_ALLOCATION; - app->heard_capacity = sources; - for (unsigned h = 0; h < sources && result == PURE_OK; ++h) { - size_t length = at < size ? bytes[at] : 0; - if (!length || length > PURE_SOURCE || size - at < 1 + length + 8) { result = PURE_STORE_ERROR; break; } - struct PureHeard *heard = &app->heard[app->heard_count++]; - memcpy(heard->source, bytes + at + 1, length); - heard->seq = heard->saved = get_le(bytes + at + 1 + length, 8); - at += 1 + length + 8; - } - while (result == PURE_OK && at < size) { - if (size - at < 13 || bytes[at] >= outputs) { result = PURE_STORE_ERROR; break; } - unsigned t = bytes[at]; - uint64_t seq = get_le(bytes + at + 1, 8); - size_t length = (size_t)get_le(bytes + at + 9, 4); - at += 13; - if (size - at < length) { result = PURE_STORE_ERROR; break; } - if (!fits[t]) { result = PURE_CONFLICT; break; } - if (app->sent_count == app->sent_capacity) { - size_t capacity = app->sent_capacity ? 2 * app->sent_capacity : 16; - struct PureSent *grown = realloc(app->sent, capacity * sizeof(*grown)); - if (!grown) { result = PURE_ALLOCATION; break; } - app->sent = grown; - app->sent_capacity = capacity; - } - unsigned char *payload = malloc(length ? length : 1); - if (!payload) { result = PURE_ALLOCATION; break; } - if (length) memcpy(payload, bytes + at, length); - app->sent[app->sent_count++] = (struct PureSent){(uint32_t)output[t], seq, payload, length, false}; - at += length; - } - free(bytes); - app->sent_durable = app->sent_count; - memcpy(app->saved_sent, app->last_sent, sizeof(app->saved_sent)); - return result; -} - -PureResult pure_save_commit(PureApp *app, const double *values, void *const *documents, size_t document_count, - int64_t *revisions) { - if (!pure_save_enabled(app)) { - /* Without a host that saves together, a save has one document at most. */ - PureResult result = PURE_OK; - for (size_t i = 0; i < document_count && result == PURE_OK; ++i) - result = app->data.commit(app->data.context, documents[i], &revisions[i]); - return result; - } - const PureModule *m = app->module; - PureDraft *drafts[2 * PURE_MODULE_STATES + 2]; - uint32_t states[2 * PURE_MODULE_STATES + 2]; - size_t count = 0; - for (uint32_t i = 0; i < m->states; ++i) { - PureDraft *draft = m->state_type[i] == PURE_STATE_TEXT ? app->texts[i] : - m->state_type[i] == PURE_STATE_LIST ? app->list_drafts[i] : NULL; - if (draft && !same_version(pure_draft_version(draft), app->saved[i])) { - states[count] = i; - drafts[count++] = draft; - } - } - /* Numbers, cursors and indices: stored again only when their bytes change. */ - PureVersion values_before = {0, 0}; - bool values_changed = false; - if (app->stored_values) { - unsigned char encoded[VALUES_BYTES], *stored; - size_t length = encode_values(m, values, encoded), size; - PureResult result = read_whole(app->stored_values, &stored, &size); - if (result != PURE_OK) return result; - values_changed = size != length || memcmp(stored, encoded, length); - free(stored); - if (values_changed) { - values_before = pure_draft_version(app->stored_values); - pure_draft_hold(app->stored_values, true); - result = pure_draft_splice(app->stored_values, 0, size, (PureSpan){encoded, length}); - if (result != PURE_OK) { pure_draft_hold(app->stored_values, false); return result; } - states[count] = UINT32_MAX; - drafts[count++] = app->stored_values; - } - } - /* The links, written whole when they changed. */ - PureVersion links_before = {0, 0}; - bool links_written = false; - if (app->links_changed) { - PureResult result = app->stored_links ? PURE_OK : open_state(app, "", "links", &app->stored_links); - size_t length; - unsigned char *encoded = result == PURE_OK ? encode_links(app, &length) : NULL; - if (result == PURE_OK && !encoded) result = PURE_ALLOCATION; - if (result == PURE_OK) { - links_before = pure_draft_version(app->stored_links); - pure_draft_hold(app->stored_links, true); - result = pure_draft_splice(app->stored_links, 0, pure_draft_size(app->stored_links), (PureSpan){encoded, length}); - if (result != PURE_OK) pure_draft_hold(app->stored_links, false); - } - free(encoded); - if (result != PURE_OK) { - if (values_changed) { - pure_draft_restore(app->stored_values, values_before); - pure_draft_hold(app->stored_values, false); - } - return result; - } - links_written = true; - states[count] = UINT32_MAX; - drafts[count++] = app->stored_links; - } - if (!count && !document_count) return PURE_OK; - PureResult result = app->data.save(app->data.context, documents, document_count, drafts, count, revisions); - for (size_t k = 0; result == PURE_OK && k < count; ++k) - if (states[k] != UINT32_MAX) app->saved[states[k]] = pure_draft_version(drafts[k]); - if (values_changed) { - if (result != PURE_OK) pure_draft_restore(app->stored_values, values_before); - pure_draft_hold(app->stored_values, false); - } - if (links_written) { - if (result != PURE_OK) pure_draft_restore(app->stored_links, links_before); - pure_draft_hold(app->stored_links, false); - } - return result; -} diff --git a/prototypes/connections/module-save.h b/prototypes/connections/module-save.h deleted file mode 100644 index bc7adcc..0000000 --- a/prototypes/connections/module-save.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef PURE_MODULE_SAVE_H -#define PURE_MODULE_SAVE_H -#include "module.h" - -/* App states a data host keeps (docs/design/008-module-format.md), internal - * to the module executor. Each text state is a stored draft, each list state - * a stored draft its list mirrors, and numbers, cursors and indices are one - * stored value by name. A save publishes what changed, all or nothing. */ - -bool pure_save_enabled(const PureApp *); -/* A text state of m: its stored value, or its initial constant when none. */ -PureResult pure_save_open_text(PureApp *, const PureModule *m, uint32_t state, PureDraft **out, PureVersion *saved); -/* A list state of m: its stored items, fields moved by name when the stored - * record differs from m's. */ -PureResult pure_save_open_list(PureApp *, const PureModule *m, uint32_t state, PureList **out, PureDraft **draft, - PureVersion *saved); -/* Open the stored numbers, cursors and indices, and put each stored one of the - * same name and kind that still fits into values, for the states of m that - * kept marks as new (-1); texts and lists are m's. */ -PureResult pure_save_open_values(PureApp *, const PureModule *m, double *values, const int *kept, - PureDraft *const *texts, PureList *const *lists); -/* Store a list whose record changed in its draft, header and items anew. */ -PureResult pure_save_rewrite_list(const PureModule *m, uint32_t state, PureList *list, PureDraft *draft); -/* The stored links of a starting app (module.h, PureApp); CONFLICT when an - * output with sends left is missing or has other fields. */ -PureResult pure_save_open_links(PureApp *, const PureModule *m); -/* Publish the documents and every state changed since the last save - * (values are the states' numbers); revisions are the documents'. */ -PureResult pure_save_commit(PureApp *, const double *values, void *const *documents, size_t document_count, - int64_t *revisions); - -/* A list with record's fields, taking from `from` each field of the same name - * and kind and giving the others their defaults (module.c). */ -PureResult pure_list_migrate(PureList **out, const PureList *from, const char *const *names, - const uint8_t *text, unsigned count, const PureModule *m, uint32_t record); -#endif diff --git a/prototypes/connections/module-store.c b/prototypes/connections/module-store.c deleted file mode 100644 index 7e759fd..0000000 --- a/prototypes/connections/module-store.c +++ /dev/null @@ -1,88 +0,0 @@ -#include "module-store.h" - -static PureResult open_document(void *context, uint32_t collection, const char *key, - void **document, int64_t *revision) { - (void)collection; /* v2 modules have one collection: the store's key space */ - PureDraft *draft; - PureResult result = pure_draft_open(context, key, &draft); - if (result != PURE_OK) return result; - pure_draft_hold(draft, true); /* values read in the event stay readable */ - *document = draft; - *revision = pure_draft_revision(draft); - return PURE_OK; -} - -static PureVersion version(void *context, void *document) { - (void)context; - return pure_draft_version(document); -} - -static PureResult read(void *context, void *document, PureVersion at, size_t offset, size_t length, - bool (*visit)(void *, PureSpan), void *visit_context) { - (void)context; - return pure_draft_read_version(document, at, offset, length, visit, visit_context); -} - -static PureResult splice(void *context, void *document, size_t offset, size_t remove, PureSpan insert) { - (void)context; - return pure_draft_splice(document, offset, remove, insert); -} - -static PureResult commit(void *context, void *document, int64_t *revision) { - (void)context; - return pure_draft_commit(document, revision); -} - -static void close_document(void *context, void *document) { - (void)context; - pure_draft_close(document); -} - -static PureResult state(void *context, const char *name, PureDraft **out) { - return pure_state_open(context, name, out); -} - -enum { SAVED = PURE_APP_PENDING + 2 * PURE_MODULE_STATES + 1 }; - -/* The documents and the app's changed states in one tree commit. */ -static PureResult save(void *context, void *const *documents, size_t document_count, PureDraft *const *states, - size_t count, int64_t *revisions) { - (void)context; - PureDraft *drafts[SAVED] = {0}; - int64_t all[SAVED] = {0}; - if (document_count > PURE_APP_PENDING || count > SAVED - document_count) return PURE_LIMIT; - for (size_t i = 0; i < document_count; ++i) drafts[i] = documents[i]; - for (size_t i = 0; i < count; ++i) drafts[document_count + i] = states[i]; - PureResult result = pure_draft_commit_all(drafts, document_count + count, all); - for (size_t i = 0; i < document_count; ++i) revisions[i] = all[i]; - return result; -} - -/* Through a store handle: documents and states share its mapping. */ -static PureResult open_handle_document(void *context, uint32_t collection, const char *key, void **document, - int64_t *revision) { - (void)collection; - PureDraft *draft; - PureResult result = pure_store_draft(context, key, &draft); - if (result != PURE_OK) return result; - pure_draft_hold(draft, true); - *document = draft; - *revision = pure_draft_revision(draft); - return PURE_OK; -} - -static PureResult handle_state(void *context, const char *name, PureDraft **out) { - return pure_store_state(context, name, out); -} - -PureData pure_store_data(PureStore *store) { - return (PureData){.context = store, .open = open_handle_document, .version = version, .read = read, - .splice = splice, .commit = commit, .close = close_document, .state = handle_state, - .save = save}; -} - -PureData pure_tree_data(const char *path) { - return (PureData){.context = (void *)path, .open = open_document, .version = version, - .read = read, .splice = splice, .commit = commit, .close = close_document, - .state = state, .save = save}; -} diff --git a/prototypes/connections/module-store.h b/prototypes/connections/module-store.h deleted file mode 100644 index d14ab3d..0000000 --- a/prototypes/connections/module-store.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef PURE_MODULE_STORE_H -#define PURE_MODULE_STORE_H -#include "module.h" -/* Keyed module collections and the app's states on the selected tree store: - * one event opens the message key as a held draft and commits it together - * with the states the event changed. The path outlives the app. */ -PureData pure_tree_data(const char *path); -/* The same through a store handle (state.h): opening a document or state - * checks only whether the store changed. The handle outlives the app. */ -PureData pure_store_data(PureStore *); -#endif diff --git a/prototypes/connections/module-text.c b/prototypes/connections/module-text.c deleted file mode 100644 index 68c03b0..0000000 --- a/prototypes/connections/module-text.c +++ /dev/null @@ -1,189 +0,0 @@ -#include "module-text.h" -#include -#include -#include -#include - -enum { CHUNK = 4096 }; - -/* state: pending continuation bytes (bits 0-1) and the allowed range of the - * next one (bits 2-4): 0 any, 1 A0-BF after E0, 2 80-9F after ED, 3 90-BF - * after F0, 4 80-8F after F4. */ -bool pure_utf8_step(unsigned *state, PureSpan span) { - unsigned remaining = *state & 3, range = *state >> 2; - for (size_t i = 0; i < span.size; ++i) { - /* Eight ASCII bytes at a time between characters. */ - uint64_t word; - while (!remaining && span.size - i >= 8 && (memcpy(&word, span.data + i, 8), !(word & UINT64_C(0x8080808080808080)))) - i += 8; - if (i == span.size) break; - unsigned char b = span.data[i]; - if (remaining) { - unsigned low = range == 1 ? 0xA0 : range == 3 ? 0x90 : 0x80; - unsigned high = range == 2 ? 0x9F : range == 4 ? 0x8F : 0xBF; - if (b < low || b > high) return false; - --remaining; - range = 0; - } else if (b < 0x80) { - continue; - } else if (b >= 0xC2 && b <= 0xDF) { - remaining = 1; - } else if (b >= 0xE0 && b <= 0xEF) { - remaining = 2; - range = b == 0xE0 ? 1 : b == 0xED ? 2 : 0; - } else if (b >= 0xF0 && b <= 0xF4) { - remaining = 3; - range = b == 0xF0 ? 3 : b == 0xF4 ? 4 : 0; - } else return false; - } - *state = remaining | range << 2; - return true; -} - -bool pure_utf8_complete(unsigned state) { return (state & 3) == 0; } - -bool pure_utf8_valid(PureSpan span) { - unsigned state = 0; - return pure_utf8_step(&state, span) && pure_utf8_complete(state); -} - -size_t pure_utf8_scalars(PureSpan span) { - size_t count = 0; - for (size_t i = 0; i < span.size; ++i) count += (span.data[i] & 0xC0) != 0x80; - return count; -} - -size_t pure_utf8_newlines(PureSpan span) { - size_t count = 0; - for (size_t i = 0; i < span.size; ++i) count += span.data[i] == '\n'; - return count; -} - -size_t pure_format_number(double value, char out[32]) { - if (value == 0) { memcpy(out, "0", 2); return 1; } - if (value == floor(value) && fabs(value) < 1e15) { - /* An integer's digits, without printf (the common case, many times cheaper). */ - char digits[16]; - uint64_t n = (uint64_t)fabs(value); - size_t count = 0, length = 0; - do { digits[count++] = (char)('0' + n % 10); n /= 10; } while (n); - if (value < 0) out[length++] = '-'; - while (count) out[length++] = digits[--count]; - out[length] = 0; - return length; - } - for (int precision = 1; precision <= 17; ++precision) { - snprintf(out, 32, "%.*g", precision, value); - if (strtod(out, NULL) == value) break; - } - char *exponent = strchr(out, 'e'); - if (exponent) { - char *digits = exponent + 2; /* after the sign */ - while (digits[0] == '0' && digits[1]) memmove(digits, digits + 1, strlen(digits)); - } - return strlen(out); -} - -typedef struct { unsigned char *out; } Copy; - -static bool copy(void *context, PureSpan span) { - Copy *c = context; - memcpy(c->out, span.data, span.size); - c->out += span.size; - return true; -} - -static void read_bytes(const PureDraft *draft, PureVersion version, size_t offset, size_t length, - unsigned char *out) { - Copy c = {out}; - if (length) pure_draft_read_version(draft, version, offset, length, copy, &c); -} - -static size_t forward(const PureDraft *draft, PureVersion version, size_t position) { - if (position >= version.bytes) return version.bytes; - unsigned char lead; - read_bytes(draft, version, position, 1, &lead); - size_t length = lead < 0x80 ? 1 : lead < 0xE0 ? 2 : lead < 0xF0 ? 3 : 4; - return position + length <= version.bytes ? position + length : version.bytes; -} - -static size_t backward(const PureDraft *draft, PureVersion version, size_t position) { - if (!position) return 0; - size_t take = position < 4 ? position : 4; - unsigned char bytes[4]; - read_bytes(draft, version, position - take, take, bytes); - size_t at = take - 1; - while (at && (bytes[at] & 0xC0) == 0x80) --at; - return position - take + at; -} - -/* Saturating: stops at the start or end, so work is bounded by the text size. */ -size_t pure_text_step(const PureDraft *draft, PureVersion version, size_t position, double count) { - for (; count > 0 && position < version.bytes; --count) position = forward(draft, version, position); - for (; count < 0 && position > 0; ++count) position = backward(draft, version, position); - return position; -} - -size_t pure_text_line_start(const PureDraft *draft, PureVersion version, size_t position) { - unsigned char chunk[CHUNK]; - while (position) { - size_t take = position < CHUNK ? position : CHUNK; - read_bytes(draft, version, position - take, take, chunk); - for (size_t i = take; i-- > 0;) - if (chunk[i] == '\n') return position - take + i + 1; - position -= take; - } - return 0; -} - -size_t pure_text_line_end(const PureDraft *draft, PureVersion version, size_t position) { - unsigned char chunk[CHUNK]; - while (position < version.bytes) { - size_t take = version.bytes - position < CHUNK ? version.bytes - position : CHUNK; - read_bytes(draft, version, position, take, chunk); - for (size_t i = 0; i < take; ++i) - if (chunk[i] == '\n') return position + i; - position += take; - } - return version.bytes; -} - -static size_t advance_within(const PureDraft *draft, PureVersion version, size_t position, - size_t columns, size_t limit) { - for (; columns && position < limit; --columns) position = forward(draft, version, position); - return position < limit ? position : limit; -} - -/* Moves keep the column; with no further line the cursor goes to the end or - * the start of the text. */ -size_t pure_text_line_step(const PureDraft *draft, PureVersion version, size_t position, double count) { - size_t start = pure_text_line_start(draft, version, position), column = 0; - for (size_t at = start; at < position; at = forward(draft, version, at)) ++column; - for (; count > 0; --count) { - size_t end = pure_text_line_end(draft, version, position); - if (end == version.bytes) return version.bytes; - size_t next = end + 1; - position = advance_within(draft, version, next, column, pure_text_line_end(draft, version, next)); - } - for (; count < 0; ++count) { - size_t here = pure_text_line_start(draft, version, position); - if (!here) return 0; - size_t previous = pure_text_line_start(draft, version, here - 1); - position = advance_within(draft, version, previous, column, here - 1); - } - return position; -} - -typedef struct { size_t count; unsigned unit; } Count; - -static bool count_span(void *context, PureSpan span) { - Count *c = context; - c->count += c->unit ? pure_utf8_newlines(span) : pure_utf8_scalars(span); - return true; -} - -size_t pure_text_count(const PureDraft *draft, PureVersion version, size_t position, unsigned unit) { - Count c = {0, unit}; - if (position) pure_draft_read_version(draft, version, 0, position, count_span, &c); - return c.count + (unit ? 1 : 0); -} diff --git a/prototypes/connections/module-text.h b/prototypes/connections/module-text.h deleted file mode 100644 index 4843928..0000000 --- a/prototypes/connections/module-text.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef PURE_MODULE_TEXT_H -#define PURE_MODULE_TEXT_H -#include "flow.h" -#include "state.h" -#include -#include - -/* Text helpers of the module executor. Text is valid UTF-8; a character is a - * Unicode scalar value, so every position these return is a scalar boundary. */ - -/* Strict UTF-8: no overlongs, surrogates or values past U+10FFFF. state - * carries a sequence split across spans; start it at 0 and finish with - * pure_utf8_complete. */ -bool pure_utf8_step(unsigned *state, PureSpan span); -bool pure_utf8_complete(unsigned state); -bool pure_utf8_valid(PureSpan span); - -/* Scalars (bytes that are not 10xxxxxx) and newlines in a span. */ -size_t pure_utf8_scalars(PureSpan span); -size_t pure_utf8_newlines(PureSpan span); - -/* Shortest decimal that reads back to the same finite double: integers - * below 1e15 without a point or exponent, otherwise %g form with a trimmed - * exponent. -0 prints as 0. Returns the length written (at most 32). */ -size_t pure_format_number(double value, char out[32]); - -/* Positions in one version of a text draft (size is version.bytes). */ -size_t pure_text_step(const PureDraft *, PureVersion, size_t position, double count); -size_t pure_text_line_step(const PureDraft *, PureVersion, size_t position, double count); -size_t pure_text_line_start(const PureDraft *, PureVersion, size_t position); -size_t pure_text_line_end(const PureDraft *, PureVersion, size_t position); -/* Scalars (unit 0) or line number counting from 1 (unit 1) before position. */ -size_t pure_text_count(const PureDraft *, PureVersion, size_t position, unsigned unit); -#endif diff --git a/prototypes/connections/module.c b/prototypes/connections/module.c deleted file mode 100644 index 697e918..0000000 --- a/prototypes/connections/module.c +++ /dev/null @@ -1,2122 +0,0 @@ -#include "module.h" -#include "module-save.h" -#include "module-text.h" -#include -#include -#include -#include - -enum { STATE_BYTES = 16, COLLECTION_BYTES = 12, INPUT_BYTES = 16, FIELD_BYTES = 8, - CONSTANT_BYTES = 8, RECORD_BYTES = 8, RECORD_FIELD_BYTES = 16, VIEW_BYTES = 16, - HANDLER_BYTES = 12, OP_BYTES = 16 }; -/* T_FIELD and T_ITEM stand for the type of the message or record field - * named, T_KEY for the type of the named keyed list's key. */ -enum { T_NUM = 1, T_BYTES = 2, T_TEXT = 3, T_CURSOR = 4, T_FIELD = 5, T_INDEX = 6, T_ITEM = 7, T_KEY = 8 }; -enum { FROM_PAYLOAD, FROM_DOCUMENT, FROM_COPY, FROM_CONSTANT, FROM_TEXT, FROM_ARENA }; -/* Where an opcode may appear. */ -enum { IN_VIEW = 1, IN_HANDLER = 2, IN_MESSAGE = 4, IN_KEYED = 8, IN_TEXT = 16, IN_ROW = 32 }; -enum { READ = IN_VIEW | IN_HANDLER }; -/* What an argument names: nothing (it is 0), a state of one type (A_STATE + - * the type), a list and one of its record's fields, a message field, the - * collection, a constant, a unit, the column's row, a branch target, a stack - * depth, a fold, a relation or a keyed list. */ -enum { A_NONE, A_STATE, A_ITEM = A_STATE + PURE_STATE_INDEX + 1, A_FIELD, A_COLLECTION, A_CONSTANT, A_UNIT, - A_ROW, A_TARGET, A_DEPTH, A_FOLD, A_RELATION, A_KEYED, A_OUTPUT }; -/* A fold's block is verified as a column that leaves one number (count) or - * two (a value and whether to include it), a table's filter as a column that - * leaves one: these stand for a view type. */ -enum { FOLD_ONE = 100, FOLD_TWO, FILTER }; -/* The text or list an opcode works on: its cursor and index operands belong - * to it, and so does a cursor or index it pushes. O_ANY: no rule. O_OPERAND: - * it pushes a position in its operand's text or list. O_ARG: the argument. - * O_REF: the text or list of the argument's cursor or index state. O_LIST: - * the list of an item argument. */ -enum { O_ANY, O_OPERAND, O_ARG, O_REF, O_LIST }; -static const unsigned char magic[8] = {'P','U','R','E','M','O','D','5'}; - -/* Operand types popped (bottom to top), the type pushed, where allowed, what - * the argument names and the text or list worked on. */ -#define NUMS(pops) {pops, {T_NUM, T_NUM, T_NUM}, T_NUM, READ, A_NONE, O_ANY} -#define STATE(type) A_STATE + PURE_STATE_##type -static const struct { unsigned char pops, in[3], out, where, arg, of; } signature[PURE_OP_COUNT] = { - [PURE_OP_CONST] = NUMS(0), [PURE_OP_ADD] = NUMS(2), [PURE_OP_SUB] = NUMS(2), [PURE_OP_MUL] = NUMS(2), - [PURE_OP_DIV] = NUMS(2), [PURE_OP_NEG] = NUMS(1), [PURE_OP_LT] = NUMS(2), [PURE_OP_LE] = NUMS(2), - [PURE_OP_GT] = NUMS(2), [PURE_OP_GE] = NUMS(2), [PURE_OP_EQ] = NUMS(2), [PURE_OP_NE] = NUMS(2), - [PURE_OP_AND] = NUMS(2), [PURE_OP_OR] = NUMS(2), [PURE_OP_NOT] = NUMS(1), [PURE_OP_MIN] = NUMS(2), - [PURE_OP_MAX] = NUMS(2), [PURE_OP_CLAMP] = NUMS(3), - [PURE_OP_LOAD] = {0, {0}, T_NUM, READ, STATE(NUM), O_ANY}, - [PURE_OP_DT] = {0, {0}, T_NUM, IN_HANDLER, A_NONE, O_ANY}, - [PURE_OP_STORE] = {1, {T_NUM}, 0, IN_HANDLER, STATE(NUM), O_ANY}, - [PURE_OP_JZ] = {1, {T_NUM}, 0, READ, A_TARGET, O_ANY}, - [PURE_OP_JMP] = {0, {0}, 0, READ, A_TARGET, O_ANY}, - [PURE_OP_FIELD] = {0, {0}, T_FIELD, IN_MESSAGE, A_FIELD, O_ANY}, - [PURE_OP_DOC] = {0, {0}, T_BYTES, IN_KEYED, A_COLLECTION, O_ANY}, - [PURE_OP_DOC_SIZE] = {0, {0}, T_NUM, IN_KEYED, A_COLLECTION, O_ANY}, - [PURE_OP_LEN] = {1, {T_BYTES}, T_NUM, IN_HANDLER, A_NONE, O_ANY}, - [PURE_OP_REPLACE] = {3, {T_NUM, T_NUM, T_BYTES}, 0, IN_KEYED, A_COLLECTION, O_ANY}, - [PURE_OP_REPLY] = {1, {T_BYTES}, 0, IN_MESSAGE, A_NONE, O_ANY}, - [PURE_OP_REQUIRE] = {1, {T_NUM}, 0, IN_HANDLER, A_NONE, O_ANY}, - [PURE_OP_TEXT_CONST] = {0, {0}, T_TEXT, READ, A_CONSTANT, O_ANY}, - [PURE_OP_TEXT_LOAD] = {0, {0}, T_TEXT, READ, STATE(TEXT), O_ANY}, - [PURE_OP_TEXT_INPUT] = {0, {0}, T_TEXT, IN_TEXT, A_NONE, O_ANY}, - [PURE_OP_NUM_TEXT] = {1, {T_NUM}, T_TEXT, READ, A_NONE, O_ANY}, - [PURE_OP_CONCAT] = {2, {T_TEXT, T_TEXT}, T_TEXT, READ, A_NONE, O_ANY}, - [PURE_OP_CHARS] = {1, {T_TEXT}, T_NUM, READ, A_NONE, O_ANY}, - [PURE_OP_LINES] = {1, {T_TEXT}, T_NUM, READ, A_NONE, O_ANY}, - [PURE_OP_TEXT_BYTES] = {1, {T_TEXT}, T_NUM, READ, A_NONE, O_ANY}, - [PURE_OP_CUR_LOAD] = {0, {0}, T_CURSOR, READ, STATE(CURSOR), O_REF}, - [PURE_OP_CUR_START] = {0, {0}, T_CURSOR, READ, STATE(TEXT), O_ARG}, - [PURE_OP_CUR_END] = {0, {0}, T_CURSOR, READ, STATE(TEXT), O_ARG}, - [PURE_OP_CUR_MOVE] = {2, {T_CURSOR, T_NUM}, T_CURSOR, READ, A_UNIT, O_OPERAND}, - [PURE_OP_CUR_STORE] = {1, {T_CURSOR}, 0, IN_HANDLER, STATE(CURSOR), O_REF}, - [PURE_OP_TEXT_INSERT] = {2, {T_CURSOR, T_TEXT}, 0, IN_HANDLER, STATE(TEXT), O_ARG}, - [PURE_OP_TEXT_REMOVE] = {2, {T_CURSOR, T_CURSOR}, 0, IN_HANDLER, STATE(TEXT), O_ARG}, - [PURE_OP_TEXT_LINE] = {1, {T_CURSOR}, T_TEXT, READ, STATE(TEXT), O_ARG}, - [PURE_OP_TEXT_SET] = {1, {T_TEXT}, 0, IN_HANDLER, STATE(TEXT), O_ANY}, - [PURE_OP_TEXT_FROM] = {1, {T_BYTES}, T_TEXT, READ, A_NONE, O_ANY}, - [PURE_OP_BYTES_FROM] = {1, {T_TEXT}, T_BYTES, READ, A_NONE, O_ANY}, - [PURE_OP_CUR_COUNT] = {1, {T_CURSOR}, T_NUM, READ, A_UNIT, O_ANY}, - /* Record operations also take as many values as the list's record has fields. */ - [PURE_OP_LIST_COUNT] = {0, {0}, T_NUM, READ, STATE(LIST), O_ANY}, - [PURE_OP_LIST_APPEND] = {0, {0}, 0, IN_HANDLER, STATE(LIST), O_ANY}, - [PURE_OP_LIST_INSERT] = {1, {T_INDEX}, 0, IN_HANDLER, STATE(LIST), O_ARG}, - [PURE_OP_LIST_REMOVE] = {1, {T_INDEX}, 0, IN_HANDLER, STATE(LIST), O_ARG}, - [PURE_OP_ITEM_GET] = {1, {T_INDEX}, T_ITEM, READ, A_ITEM, O_LIST}, - [PURE_OP_ITEM_SET] = {1, {T_INDEX}, 0, IN_HANDLER, A_ITEM, O_LIST}, - [PURE_OP_INDEX_LOAD] = {0, {0}, T_INDEX, READ, STATE(INDEX), O_REF}, - [PURE_OP_INDEX_STORE] = {1, {T_INDEX}, 0, IN_HANDLER, STATE(INDEX), O_REF}, - [PURE_OP_INDEX_MOVE] = {2, {T_INDEX, T_NUM}, T_INDEX, READ, A_NONE, O_OPERAND}, - [PURE_OP_INDEX_FIRST] = {0, {0}, T_INDEX, READ, STATE(LIST), O_ARG}, - [PURE_OP_INDEX_LAST] = {0, {0}, T_INDEX, READ, STATE(LIST), O_ARG}, - [PURE_OP_INDEX_NUM] = {1, {T_INDEX}, T_NUM, READ, A_NONE, O_ANY}, - [PURE_OP_ROW] = {0, {0}, T_INDEX, IN_ROW, A_ROW, O_ARG}, - /* Stack copies: their types come from the values they move. */ - [PURE_OP_PICK] = {0, {0}, 0, READ, A_DEPTH, O_ANY}, - [PURE_OP_SLIDE] = {0, {0}, 0, READ, A_DEPTH, O_ANY}, - [PURE_OP_FOLD] = {0, {0}, T_NUM, READ, A_FOLD, O_ANY}, - [PURE_OP_TEXT_COMPARE] = {2, {T_TEXT, T_TEXT}, T_NUM, READ, A_RELATION, O_ANY}, - [PURE_OP_LIST_FIND] = {1, {T_KEY}, T_INDEX, READ, A_KEYED, O_ARG}, - [PURE_OP_LIST_HAS] = {1, {T_KEY}, T_NUM, READ, A_KEYED, O_ANY}, - [PURE_OP_NOW] = {0, {0}, T_NUM, IN_HANDLER, A_NONE, O_ANY}, - [PURE_OP_SEND] = {0, {0}, 0, IN_HANDLER, A_OUTPUT, O_ANY}, - [PURE_OP_FLOOR] = NUMS(1), [PURE_OP_MOD] = NUMS(2), -}; -#undef NUMS -#undef STATE - -static uint32_t read16(const unsigned char *p) { return (uint32_t)p[0] | (uint32_t)p[1] << 8; } - -static uint32_t read32(const unsigned char *p) { - return (uint32_t)p[0] | (uint32_t)p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; -} - -static void write32(unsigned char *p, uint32_t value) { - for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(value >> (8 * i)); -} - -static double read_double(const unsigned char *p) { - uint64_t bits = read32(p) | (uint64_t)read32(p + 4) << 32; - double value; - memcpy(&value, &bits, sizeof(value)); - return value; -} - -PURE_COLD static bool key_name(const char *key) { - static const char *const named[] = {"Space", "Enter", "ArrowLeft", "ArrowRight", "ArrowUp", "ArrowDown", - "Backspace", "Delete", "Home", "End", "Tab", "Escape", "PageUp", "PageDown"}; - for (size_t i = 0; i < sizeof(named) / sizeof(*named); ++i) - if (!strcmp(key, named[i])) return true; - if (!strncmp(key, "Key", 3)) return key[3] >= 'A' && key[3] <= 'Z' && !key[4]; - if (!strncmp(key, "Digit", 5)) return key[5] >= '0' && key[5] <= '9' && !key[6]; - return false; -} - -/* A document key: 1..64 bytes of [A-Za-z0-9_-], as every Pure store accepts. */ -static bool document_key(const char *key) { - size_t length = key ? strlen(key) : 0; - if (!length || length > 64) return false; - for (size_t i = 0; i < length; ++i) { - char c = key[i]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || - c == '-' || c == '_')) return false; - } - return true; -} - -/* Strings are [u8 length][bytes]; a name is 1..31 bytes: an ASCII identifier, - * a path of identifiers joined by dots (a message field inside a group, as - * user.name), or any printable ASCII (a key's name). */ -enum { NAME_IDENTIFIER, NAME_PATH, NAME_PRINTABLE }; -PURE_COLD static bool name_at(const unsigned char *strings, uint32_t size, uint32_t offset, - int kind, char out[PURE_MODULE_NAME]) { - if (offset >= size) return false; - uint32_t length = strings[offset]; - if (!length || length >= PURE_MODULE_NAME || length > size - offset - 1) return false; - for (uint32_t i = 0, start = 0; i < length; ++i) { - unsigned char c = strings[offset + 1 + i]; - bool letter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; - if (kind == NAME_PATH && c == '.' && i > start && i + 1 < length) { start = i + 1; continue; } - if (!letter && !(i > start && c >= '0' && c <= '9') && (kind != NAME_PRINTABLE || c < 32 || c > 126)) - return false; - } - memcpy(out, strings + offset + 1, length); - out[length] = 0; - return true; -} - -/* The input whose handler this is, or -1 for frame. */ -static int event_input(uint32_t event) { return event ? (int)((event - 1) / 2) : -1; } - -/* The state an argument names must have this type. */ -static bool state_of(const PureModule *m, uint32_t state, uint8_t type) { - return state < m->states && m->state_type[state] == type; -} - -/* The list an item operation names: its low 16 bits (the field is above). */ -static uint32_t list_of(uint32_t arg) { return arg & 0xFFFF; } -static uint32_t item_field(uint32_t arg) { return arg >> 16; } - -static unsigned char field_type(const PureModule *m, uint32_t list, uint32_t field) { - return m->record_field_text[m->record[m->state_ref[list]].first + field] ? T_TEXT : T_NUM; -} - -PURE_COLD static bool names_arg(const PureModule *m, unsigned kind, uint32_t arg, const PureInputShape *shape, int row) { - switch (kind) { - case A_ITEM: - return state_of(m, list_of(arg), PURE_STATE_LIST) && item_field(arg) < m->record[m->state_ref[list_of(arg)]].count; - case A_FIELD: return arg < shape->field_count; - case A_COLLECTION: return arg < m->collections; - case A_CONSTANT: return arg < m->constants; - case A_UNIT: return arg <= PURE_UNIT_LINE; - case A_ROW: return (int)arg == row; - case A_FOLD: return (arg & 0xFF) <= PURE_FOLD_FIRST && state_of(m, arg >> 8 & 0xFF, PURE_STATE_LIST) && arg >> 16; - case A_RELATION: return arg <= PURE_NOT_EQUAL; - case A_KEYED: return state_of(m, arg, PURE_STATE_LIST) && m->state_key[arg]; - case A_OUTPUT: return arg < m->inputs && m->input[arg].kind == PURE_KIND_OUTPUT; - case A_NONE: case A_TARGET: case A_DEPTH: return true; - default: return state_of(m, arg, (uint8_t)(kind - A_STATE)); - } -} - -/* The stack a branch brings to its target, while the target is ahead. */ -typedef struct { uint32_t target; int height; unsigned char types[PURE_MODULE_STACK], owners[PURE_MODULE_STACK]; } Join; - -static bool same_stack(const Join *join, int height, const unsigned char *types, const unsigned char *owners) { - return join->height == height && !memcmp(join->types, types, (size_t)height) && - !memcmp(join->owners, owners, (size_t)height); -} - -/* Types and heights are checked per instruction; a cursor's type records - * which text it belongs to and an index's type which list (other values - * belong to nothing). Branches go forward within the block. Every path into - * a join brings the same stack (types and owners), so a value may cross a - * join, as in a conditional expression; at most 8 joins with a non-empty - * stack are ahead at once. Handlers end empty, views with one value of their - * type. A column (row is its list) may read the row it is evaluated for. - * A fold's block is verified as a column of its list (reads only, no fold - * in it); the block around treats it as one instruction and may not branch - * into it. Returns why the block is refused, with at moved to the - * instruction when one is at fault, or NULL. */ -PURE_COLD static const char *verify_block(const PureModule *, PureBlock, int view, int row, uint32_t event, uint32_t *at); - -/* target marks, per instruction of the block, a branch into it with an empty stack. */ -PURE_COLD static const char *verify_code(const PureModule *m, PureBlock block, int view, int row, uint32_t event, - uint32_t *at, unsigned char *target) { -#define REFUSE(why) do { *at = pc; return why; } while (0) - if (view >= 0 && !block.length) return "empty view"; - int input = view >= 0 ? -1 : event_input(event); - const PureInputShape *shape = input >= 0 ? &m->input[input] : NULL; - unsigned where = view >= 0 ? IN_VIEW | (row >= 0 ? IN_ROW : 0) : IN_HANDLER; - if (shape && shape->kind == PURE_KIND_MESSAGE) where |= IN_MESSAGE | (shape->keyed ? IN_KEYED : 0); - if (shape && shape->kind == PURE_KIND_TEXT) where |= IN_TEXT; - unsigned char types[PURE_MODULE_STACK], owners[PURE_MODULE_STACK]; - Join joins[PURE_MODULE_JOINS]; - unsigned open = 0; - int height = 0; - bool reachable = true; - for (uint32_t i = 0; i <= block.length; ++i) { - uint32_t pc = block.start + i; - /* Branches here brought an empty stack (target) or the stack of a join. */ - unsigned join = open; - for (unsigned k = 0; k < open; ++k) if (joins[k].target == pc) join = k; - if (join < open) { - if (target[i] || (reachable && !same_stack(&joins[join], height, types, owners))) - REFUSE("join with different stacks"); - height = joins[join].height; - memcpy(types, joins[join].types, (size_t)height); - memcpy(owners, joins[join].owners, (size_t)height); - joins[join] = joins[--open]; - reachable = true; - } else if (target[i]) { - if (reachable && height) REFUSE("join with different stacks"); - height = 0; - reachable = true; - } - if (i == block.length) break; - if (!reachable) REFUSE("unreachable instruction"); - const PureOp *op = &m->code[pc]; - if (op->op >= PURE_OP_COUNT) REFUSE("unknown opcode"); - unsigned allowed = signature[op->op].where, kind = signature[op->op].arg, of = signature[op->op].of; - if (!(allowed & where)) { - if (allowed & IN_ROW) REFUSE("row outside a column"); - if (view == FILTER) REFUSE("filter with an effect"); - if (view >= FOLD_ONE) REFUSE("fold with an effect"); - if (view >= 0) REFUSE("view with an effect"); - if (allowed & IN_KEYED) REFUSE("document operation outside a keyed message handler"); - if (allowed & IN_TEXT) REFUSE("text input outside a text handler"); - REFUSE("message operation outside a message handler"); - } - uint32_t arg = op->arg; - if ((kind == A_NONE && arg) || (op->op == PURE_OP_CONST ? !isfinite(op->value) : op->value != 0)) - REFUSE("non-canonical operand"); - if (!names_arg(m, kind, arg, shape, row)) REFUSE("operand names no such state, field, constant or unit"); - /* A keyed list places its items by key, and an item's key stays. */ - if (op->op == PURE_OP_LIST_INSERT && m->state_key[arg]) REFUSE("insert by position into a keyed list"); - if (op->op == PURE_OP_ITEM_SET && m->state_key[list_of(arg)] == item_field(arg) + 1) - REFUSE("change of a keyed list's key"); - /* PICK copies the value arg below the top; SLIDE drops arg values under the top. */ - if (kind == A_DEPTH) { - bool pick = op->op == PURE_OP_PICK; - if (pick ? arg >= (uint32_t)height : !arg || arg >= (uint32_t)height) REFUSE("stack underflow"); - if (pick && height == PURE_MODULE_STACK) REFUSE("stack overflow"); - int from = height - 1 - (int)arg, to = pick ? height++ : from; - types[to] = pick ? types[from] : types[height - 1]; - owners[to] = pick ? owners[from] : owners[height - 1]; - if (!pick) height = from + 1; - continue; - } - uint32_t skip = 0; - if (kind == A_FOLD) { - uint32_t inner = pc; - skip = arg >> 16; - if (view >= FOLD_ONE) REFUSE(view == FILTER ? "fold inside a filter" : "fold inside a fold"); - if (skip > block.length - i - 1) REFUSE("fold block outside its block"); - unsigned how = arg & 0xFF; - const char *why = verify_block(m, (PureBlock){pc + 1, skip}, - how == PURE_FOLD_COUNT || how == PURE_FOLD_FIRST ? FOLD_ONE : FOLD_TWO, - (int)(arg >> 8 & 0xFF), 0, &inner); - if (why) { *at = inner; return why; } - for (uint32_t k = i + 1; k <= i + skip; ++k) { - if (target[k]) REFUSE("branch into a fold"); - for (unsigned j = 0; j < open; ++j) if (joins[j].target == block.start + k) REFUSE("branch into a fold"); - } - } - /* Operand types, bottom to top; record operations take their fields. */ - unsigned char need[PURE_MODULE_FIELDS + 2]; - unsigned pops = signature[op->op].pops; - memcpy(need, signature[op->op].in, sizeof(signature[op->op].in)); - if (op->op == PURE_OP_SEND) { - const PureInputShape *output = &m->input[arg]; - for (unsigned f = 0; f < output->field_count; ++f) { - uint8_t type = m->field[output->field_start + f].type; - need[f] = type == PURE_FIELD_TEXT ? T_TEXT : type == PURE_FIELD_BYTES ? T_BYTES : T_NUM; - } - pops = output->field_count; - } else if (op->op == PURE_OP_LIST_APPEND || op->op == PURE_OP_LIST_INSERT) { - const PureRecord *record = &m->record[m->state_ref[arg]]; - for (unsigned f = 0; f < record->count; ++f) need[pops + f] = field_type(m, arg, f); - pops += record->count; - } else if (op->op == PURE_OP_ITEM_SET) { - need[pops++] = field_type(m, list_of(arg), item_field(arg)); - } else if (need[0] == T_KEY) { - need[0] = field_type(m, arg, m->state_key[arg] - 1u); - } - if (height < (int)pops) REFUSE("stack underflow"); - height -= (int)pops; - for (unsigned k = 0; k < pops; ++k) - if (types[height + (int)k] != need[k]) REFUSE("operand type"); - unsigned owner = of == O_ARG ? arg : of == O_REF ? m->state_ref[arg] : of == O_LIST ? list_of(arg) : 0; - for (unsigned k = 0; of >= O_ARG && k < pops; ++k) { - unsigned char type = types[height + (int)k]; - if ((type == T_CURSOR || type == T_INDEX) && owners[height + (int)k] != owner) - REFUSE(type == T_CURSOR ? "cursor of another text" : "index of another list"); - } - unsigned char out = signature[op->op].out; - if (kind == A_FOLD && (arg & 0xFF) == PURE_FOLD_FIRST) { /* an index of its list */ - out = T_INDEX; - of = O_ARG; - owner = arg >> 8 & 0xFF; - } - if (out == T_FIELD) { - uint8_t type = m->field[shape->field_start + arg].type; - out = type == PURE_FIELD_TEXT ? T_TEXT : type == PURE_FIELD_BYTES ? T_BYTES : T_NUM; - } - if (out == T_ITEM) out = field_type(m, list_of(arg), item_field(arg)); - if (out) { - if (height == PURE_MODULE_STACK) REFUSE("stack overflow"); - owners[height] = (unsigned char)(out != T_CURSOR && out != T_INDEX ? 0 : of == O_OPERAND ? owners[height] : owner); - types[height++] = out; - } - if (kind == A_TARGET) { - if (arg <= pc || arg > block.start + block.length) REFUSE("branch not forward within its block"); - join = open; - for (unsigned k = 0; k < open; ++k) if (joins[k].target == arg) join = k; - if (!height) { - if (join < open) REFUSE("join with different stacks"); - target[arg - block.start] = 1; - } else if (join < open) { - if (!same_stack(&joins[join], height, types, owners)) REFUSE("join with different stacks"); - } else { - if (target[arg - block.start]) REFUSE("join with different stacks"); - if (open == PURE_MODULE_JOINS) REFUSE("more than 8 joins ahead"); - joins[open] = (Join){.target = arg, .height = height}; - memcpy(joins[open].types, types, (size_t)height); - memcpy(joins[open++].owners, owners, (size_t)height); - } - if (op->op == PURE_OP_JMP) reachable = false; - } - i += skip; - } - if (!reachable) return "block end is unreachable"; - if (view == FILTER) return height != 1 || types[0] != T_NUM ? "filter must leave one number" : NULL; - if (view >= FOLD_ONE) { - int want = view - FOLD_ONE + 1; - return height != want || types[0] != T_NUM || types[want - 1] != T_NUM ? "fold block must leave its numbers" : NULL; - } - unsigned char result = view == PURE_VIEW_TEXT ? T_TEXT : T_NUM; - if (view >= 0 ? height != 1 || types[0] != result : height != 0) - return view >= 0 ? "view must leave one value of its type" : "handler must leave no value"; - return NULL; -#undef REFUSE -} - -PURE_COLD static const char *verify_block(const PureModule *m, PureBlock block, int view, int row, uint32_t event, - uint32_t *at) { - if (block.start > m->code_count || block.length > m->code_count - block.start) return "code range outside module"; - unsigned char *target = calloc((size_t)block.length + 1, 1); - if (!target) return "out of memory"; - const char *why = verify_code(m, block, view, row, event, at, target); - free(target); - return why; -} - -PURE_COLD PureResult pure_module_read(const char *path, PureModule *out, char *error, size_t error_size) { - FILE *file = fopen(path, "rb"); - unsigned char *bytes = NULL; - size_t size = 0, capacity = 0; - bool ok = file != NULL; - /* The largest module is some 2 MiB: code, strings and constants at their limits. */ - while (ok && !feof(file) && capacity < (8u << 20)) { - capacity = capacity ? capacity * 2 : 1 << 16; - unsigned char *grown = realloc(bytes, capacity); - if (!grown) { ok = false; break; } - bytes = grown; - size += fread(bytes + size, 1, capacity - size, file); - ok = !ferror(file); - } - ok = ok && feof(file); - if (file) fclose(file); - PureResult result = PURE_NOT_FOUND; - if (!ok) { - if (error_size) snprintf(error, error_size, "cannot read %s", path); - pure_module_release(out); - } else result = pure_module_load(bytes, size, out, error, error_size); - free(bytes); - return result; -} - -void pure_module_release(PureModule *m) { - free(m->storage); - m->storage = NULL; - m->code = NULL; - m->constant_offset = m->constant_length = NULL; - m->constant_data = NULL; -} - -PURE_COLD PureResult pure_module_load(const void *bytes, size_t size, PureModule *out, - char *error, size_t error_size) { - const unsigned char *p = bytes; - const char *why; - uint32_t bad = 0; - PureResult result = PURE_INVALID; -#define REFUSE(message, index) do { why = message; bad = (uint32_t)(index); goto refused; } while (0) - if (error_size) error[0] = 0; - pure_module_release(out); - memset(out, 0, sizeof(*out)); - if (!p || size < PURE_MODULE_HEADER || memcmp(p, magic, sizeof(magic))) REFUSE("not a Pure module v5", 0); - out->states = (uint16_t)read16(p + 8); - out->inputs = (uint16_t)read16(p + 10); - out->views = (uint16_t)read16(p + 12); - out->handlers = (uint16_t)read16(p + 14); - out->code_count = read32(p + 16); - uint32_t strings = read32(p + 20); - out->fields = (uint16_t)read16(p + 24); - out->collections = (uint16_t)read16(p + 26); - out->constants = (uint16_t)read16(p + 28); - out->records = (uint16_t)read16(p + 30); - uint32_t constant_bytes = read32(p + 32); - out->record_fields = (uint16_t)read16(p + 36); - if (read16(p + 38)) REFUSE("reserved header bytes", 38); - if (out->states > PURE_MODULE_STATES || out->inputs > PURE_MODULE_INPUTS || - out->views > PURE_MODULE_VIEWS || out->handlers > PURE_MODULE_HANDLERS || - out->code_count > PURE_MODULE_CODE || strings > PURE_MODULE_STRINGS || - out->fields > PURE_MODULE_FIELDS || out->collections > PURE_MODULE_COLLECTIONS || - out->constants > PURE_MODULE_CONSTANTS || constant_bytes > PURE_MODULE_CONSTANT_BYTES || - out->records > PURE_MODULE_RECORDS || out->record_fields > PURE_MODULE_RECORD_FIELDS) { - result = PURE_LIMIT; - REFUSE("module exceeds a limit", 8); - } - size_t expected = PURE_MODULE_HEADER + (size_t)out->states * STATE_BYTES + - (size_t)out->collections * COLLECTION_BYTES + (size_t)out->inputs * INPUT_BYTES + - (size_t)out->fields * FIELD_BYTES + (size_t)out->constants * CONSTANT_BYTES + - (size_t)out->records * RECORD_BYTES + (size_t)out->record_fields * RECORD_FIELD_BYTES + - (size_t)out->views * VIEW_BYTES + (size_t)out->handlers * HANDLER_BYTES + - (size_t)out->code_count * OP_BYTES + strings + constant_bytes; - if (size != expected) REFUSE("module size", expected); - /* The code, the constants' offsets and lengths, then their bytes, in one block. */ - size_t storage = (size_t)out->code_count * sizeof(PureOp) + (size_t)out->constants * 2 * sizeof(uint32_t) + - constant_bytes; - if (!(out->storage = malloc(storage ? storage : 1))) { - result = PURE_ALLOCATION; - REFUSE("out of memory", 0); - } - out->code = out->storage; - out->constant_offset = (uint32_t *)(out->code + out->code_count); - out->constant_length = out->constant_offset + out->constants; - out->constant_data = (unsigned char *)(out->constant_length + out->constants); - const unsigned char *at = p + PURE_MODULE_HEADER; - const unsigned char *text = p + size - constant_bytes - strings; - if (constant_bytes) memcpy(out->constant_data, p + size - constant_bytes, constant_bytes); - for (uint32_t i = 0; i < out->states; ++i, at += STATE_BYTES) { - out->state_type[i] = (uint8_t)read32(at + 4); - if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->state_names[i])) REFUSE("state name", i); - if (read32(at + 4) > PURE_STATE_INDEX) REFUSE("state type", i); - if (out->state_type[i] == PURE_STATE_NUM) { - out->initial[i] = read_double(at + 8); - if (!isfinite(out->initial[i])) REFUSE("non-finite initial value", i); - } else { - /* Flags: a mark (cursor), or a keyed list's key field + 1. */ - uint32_t flags = read32(at + 12); - bool cursor = out->state_type[i] == PURE_STATE_CURSOR, list = out->state_type[i] == PURE_STATE_LIST; - if ((flags && !(cursor && flags == PURE_CURSOR_STAYS) && !(list && flags <= PURE_RECORD_FIELDS)) || - read32(at + 8) > UINT16_MAX) REFUSE("state reference", i); - out->state_ref[i] = (uint16_t)read32(at + 8); - out->state_flags[i] = cursor ? (uint8_t)flags : 0; - out->state_key[i] = list ? (uint8_t)flags : 0; - } - } - for (uint32_t i = 0; i < out->collections; ++i, at += COLLECTION_BYTES) { - if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->collection_names[i])) REFUSE("collection name", i); - if (read32(at + 4) != 1 || read32(at + 8) != 1) REFUSE("collection must hold bytes by key", i); - } - uint32_t next_field = 0; - const unsigned char *fields = at + (size_t)out->inputs * INPUT_BYTES; - for (uint32_t i = 0; i < out->inputs; ++i, at += INPUT_BYTES) { - PureInputShape *shape = &out->input[i]; - shape->kind = at[4]; - shape->keyed = at[5]; - shape->field_start = (uint16_t)read16(at + 12); - shape->field_count = (uint16_t)read16(at + 14); - if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->input_names[i])) REFUSE("input name", i); - if (read16(at + 6) || at[5] > 1) REFUSE("input flags", i); - if (shape->kind == PURE_KIND_KEY) { - if (shape->keyed || shape->field_start || shape->field_count || - !name_at(text, strings, read32(at + 8), NAME_PRINTABLE, out->input_keys[i]) || - !key_name(out->input_keys[i])) REFUSE("input key", i); - for (uint32_t k = 0; k < i; ++k) - if (out->input[k].kind == PURE_KIND_KEY && !strcmp(out->input_keys[k], out->input_keys[i])) - REFUSE("duplicate key", i); - continue; - } - if (shape->kind == PURE_KIND_TEXT) { - if (shape->keyed || shape->field_start || shape->field_count || read32(at + 8)) - REFUSE("text input shape", i); - continue; - } - if (shape->kind == PURE_KIND_TIMER) { - shape->period = read32(at + 8); - if (shape->keyed || shape->field_start || shape->field_count || !shape->period || - shape->period > PURE_TIMER_LONGEST) REFUSE("timer input", i); - continue; - } - if ((shape->kind != PURE_KIND_MESSAGE && shape->kind != PURE_KIND_OUTPUT) || read32(at + 8) || - (shape->kind == PURE_KIND_OUTPUT && shape->keyed) || shape->field_start != next_field || - shape->field_count > out->fields - next_field) REFUSE("message shape", i); - /* The prefix holds the fixed fields and the variable fields' lengths - * in order, but the last variable field's: it takes the rest. */ - uint32_t last = UINT32_MAX; - for (uint32_t f = 0; f < shape->field_count; ++f) { - uint32_t type = read32(fields + (size_t)(next_field + f) * FIELD_BYTES + 4); - if (type == PURE_FIELD_BYTES || type == PURE_FIELD_TEXT) last = f; - } - for (uint32_t f = 0; f < shape->field_count; ++f) { - const unsigned char *entry = fields + (size_t)(next_field + f) * FIELD_BYTES; - PureField *field = &out->field[next_field + f]; - field->type = (uint8_t)read32(entry + 4); - if (!name_at(text, strings, read32(entry), NAME_PATH, field->name) || field->type < PURE_FIELD_U32 || - read32(entry + 4) > PURE_FIELD_TEXT) - REFUSE("message field", next_field + f); - /* A name is a field or a group, never both (user and user.name). */ - for (uint32_t k = 0; k < f; ++k) { - const char *a = out->field[next_field + k].name, *b = field->name; - size_t la = strlen(a), lb = strlen(b); - if (!strcmp(a, b)) REFUSE("duplicate field", next_field + f); - if ((la < lb && !memcmp(a, b, la) && b[la] == '.') || (lb < la && !memcmp(a, b, lb) && a[lb] == '.')) - REFUSE("field and group", next_field + f); - } - field->offset = f == last ? PURE_FIELD_REST : shape->prefix; - if (f != last) shape->prefix += field->type == PURE_FIELD_NUMBER ? 8 : 4; - } - next_field += shape->field_count; - } - if (next_field != out->fields) REFUSE("unused message fields", next_field); - at = fields + (size_t)out->fields * FIELD_BYTES; - for (uint32_t i = 0; i < out->constants; ++i, at += CONSTANT_BYTES) { - out->constant_offset[i] = read32(at); - out->constant_length[i] = read32(at + 4); - if (out->constant_offset[i] > constant_bytes || out->constant_length[i] > constant_bytes - out->constant_offset[i] || - !pure_utf8_valid((PureSpan){out->constant_data + out->constant_offset[i], out->constant_length[i]})) - REFUSE("text constant", i); - } - /* Records: fields in order, numbered from first; a text default is a constant. */ - const unsigned char *fields_at = at + (size_t)out->records * RECORD_BYTES; - uint32_t next_record_field = 0; - for (uint32_t i = 0; i < out->records; ++i, at += RECORD_BYTES) { - PureRecord *record = &out->record[i]; - record->first = (uint16_t)read16(at + 4); - record->count = (uint16_t)read16(at + 6); - if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->record_names[i]) || record->first != next_record_field || - !record->count || record->count > PURE_RECORD_FIELDS || record->count > out->record_fields - next_record_field) - REFUSE("record", i); - for (uint32_t f = record->first; f < record->first + record->count; ++f) { - const unsigned char *entry = fields_at + (size_t)f * RECORD_FIELD_BYTES; - uint32_t type = read32(entry + 4); - out->record_field_text[f] = type == 1; - if (!name_at(text, strings, read32(entry), NAME_IDENTIFIER, out->record_field_names[f]) || type > 1) - REFUSE("record field", f); - for (uint32_t k = record->first; k < f; ++k) - if (!strcmp(out->record_field_names[k], out->record_field_names[f])) REFUSE("duplicate record field", f); - out->record_field_default[f] = type ? read32(entry + 8) : read_double(entry + 8); - if (type ? read32(entry + 12) || read32(entry + 8) >= out->constants : !isfinite(out->record_field_default[f])) - REFUSE("record field default", f); - } - next_record_field += record->count; - } - if (next_record_field != out->record_fields) REFUSE("unused record fields", next_record_field); - at = fields_at + (size_t)out->record_fields * RECORD_FIELD_BYTES; - for (uint32_t i = 0; i < out->states; ++i) - if ((out->state_type[i] == PURE_STATE_TEXT && out->state_ref[i] >= out->constants) || - (out->state_type[i] == PURE_STATE_CURSOR && !state_of(out, out->state_ref[i], PURE_STATE_TEXT)) || - (out->state_type[i] == PURE_STATE_LIST && - (out->state_ref[i] >= out->records || out->state_key[i] > out->record[out->state_ref[i]].count)) || - (out->state_type[i] == PURE_STATE_INDEX && !state_of(out, out->state_ref[i], PURE_STATE_LIST))) - REFUSE("state reference", i); - /* A table names its list in place of code; its columns follow it. */ - for (uint32_t i = 0; i < out->views; ++i, at += VIEW_BYTES) { - if (!name_at(text, strings, read32(at), NAME_IDENTIFIER, out->view_names[i])) REFUSE("view name", i); - out->view_type[i] = at[4]; - out->view_table[i] = (uint16_t)read16(at + 6); - out->view_code[i] = (PureBlock){read32(at + 8), read32(at + 12)}; - uint32_t table = out->view_table[i]; - if (at[4] > PURE_VIEW_TABLE || (at[5] && at[4] != PURE_VIEW_TABLE) || - (table && (table > i || out->view_type[table - 1] != PURE_VIEW_TABLE || at[4] == PURE_VIEW_TABLE))) - REFUSE("view type", i); - /* A table names its list in start, or, filtered, in the byte after its - * type (list + 1) with its filter's block in start and length. */ - if (at[4] == PURE_VIEW_TABLE) { - uint32_t list = at[5] ? at[5] - 1u : out->view_code[i].start; - if (!state_of(out, list, PURE_STATE_LIST) || !at[5] != !out->view_code[i].length) REFUSE("table list", i); - out->view_list[i] = (uint16_t)list; - } else if (table) out->view_list[i] = out->view_list[table - 1]; - } - for (uint32_t i = 0; i < out->handlers; ++i, at += HANDLER_BYTES) { - uint32_t event = read32(at); - out->handler[i] = (PureHandler){event, {read32(at + 4), read32(at + 8)}}; - int input = event_input(event); - if (event > 2u * out->inputs || (input >= 0 && out->input[input].kind != PURE_KIND_KEY && !(event & 1)) || - (input >= 0 && out->input[input].kind == PURE_KIND_OUTPUT)) - REFUSE("handler event", i); - } - for (uint32_t i = 0; i < out->code_count; ++i, at += OP_BYTES) { - if (at[1] || at[2] || at[3]) REFUSE("instruction padding", i); - out->code[i] = (PureOp){.op = at[0], .arg = read32(at + 4), .value = read_double(at + 8)}; - } - /* States, records, collections, inputs and views share one namespace. */ - const char *names[PURE_MODULE_STATES + PURE_MODULE_RECORDS + PURE_MODULE_COLLECTIONS + - PURE_MODULE_INPUTS + PURE_MODULE_VIEWS]; - size_t count = 0; - for (uint32_t i = 0; i < out->states; ++i) names[count++] = out->state_names[i]; - for (uint32_t i = 0; i < out->records; ++i) names[count++] = out->record_names[i]; - for (uint32_t i = 0; i < out->collections; ++i) names[count++] = out->collection_names[i]; - for (uint32_t i = 0; i < out->inputs; ++i) names[count++] = out->input_names[i]; - for (uint32_t i = 0; i < out->views; ++i) if (!out->view_table[i]) names[count++] = out->view_names[i]; - for (size_t i = 0; i < count; ++i) - for (size_t k = 0; k < i; ++k) - if (!strcmp(names[i], names[k])) REFUSE("duplicate name", i); - /* Column names are unique within their table. */ - for (uint32_t i = 0; i < out->views; ++i) - for (uint32_t k = 0; k < i; ++k) - if (out->view_table[i] && out->view_table[k] == out->view_table[i] && - !strcmp(out->view_names[i], out->view_names[k])) REFUSE("duplicate column", i); - for (uint32_t i = 0; i < out->views; ++i) { - bad = i; - bool table = out->view_type[i] == PURE_VIEW_TABLE; - if ((!table || out->view_code[i].length) && - (why = verify_block(out, out->view_code[i], table ? FILTER : out->view_type[i], - table || out->view_table[i] ? out->view_list[i] : -1, 0, &bad))) goto refused; - } - for (uint32_t i = 0; i < out->handlers; ++i) { - bad = i; - if ((why = verify_block(out, out->handler[i].code, -1, -1, out->handler[i].event, &bad))) goto refused; - } - return PURE_OK; -refused: - if (error_size) snprintf(error, error_size, "%s (%u)", why, bad); - pure_module_release(out); - return result; -#undef REFUSE -} - -static int find(const char (*names)[PURE_MODULE_NAME], unsigned count, const char *name) { - for (unsigned i = 0; i < count; ++i) - if (!strcmp(names[i], name)) return (int)i; - return -1; -} - -int pure_module_state(const PureModule *m, const char *name) { return find(m->state_names, m->states, name); } -int pure_module_input(const PureModule *m, const char *name) { return find(m->input_names, m->inputs, name); } -int pure_module_view(const PureModule *m, const char *name) { - for (uint32_t i = 0; i < m->views; ++i) - if (!m->view_table[i] && !strcmp(m->view_names[i], name)) return (int)i; - return -1; -} - -int pure_module_position(const PureModule *m, uint32_t view) { - if (view >= m->views || m->view_type[view] != PURE_VIEW_NUM || m->view_table[view]) return -1; - const PureBlock *block = &m->view_code[view]; - const PureOp *code = &m->code[block->start]; - return block->length == 2 && code[0].op == PURE_OP_INDEX_LOAD && code[1].op == PURE_OP_INDEX_NUM ? (int)code[0].arg : -1; -} - -int pure_module_text_state(const PureModule *m, uint32_t view) { - if (view >= m->views || m->view_type[view] != PURE_VIEW_TEXT || m->view_table[view]) return -1; - const PureBlock *block = &m->view_code[view]; - return block->length == 1 && m->code[block->start].op == PURE_OP_TEXT_LOAD ? (int)m->code[block->start].arg : -1; -} - -int pure_module_column(const PureModule *m, uint32_t table, const char *name) { - for (uint32_t i = 0; i < m->views; ++i) - if (m->view_table[i] == table + 1 && !strcmp(m->view_names[i], name)) return (int)i; - return -1; -} - -/* ---- Execution ------------------------------------------------------ */ - -/* IEEE 754-2019 minimum/maximum on finite values: -0 orders below +0. */ -static double minimum(double a, double b) { return a < b || (a == b && signbit(a)) ? a : b; } -static double maximum(double a, double b) { return a > b || (a == b && !signbit(a)) ? a : b; } - -typedef struct { double num; PureBytes bytes; } Slot; -typedef struct { - PureApp *app; - const PureInputShape *shape; /* message, or NULL */ - const char *key; /* a keyed message's key */ - int64_t base; /* the revision a message's edit was made against, or -1 */ - PureArena *arena; - double dt; - size_t row; /* the row a column is evaluated for */ - bool modified, exists; - uint32_t (*spans)[2]; /* a message's variable fields: start and size in the payload */ -} Event; - -static PureResult fault(PureApp *app, PureFault kind, uint32_t pc) { - app->fault = kind; - app->fault_pc = pc; - return kind == PURE_FAULT_LIMIT ? PURE_LIMIT : kind == PURE_FAULT_NOT_FOUND ? PURE_NOT_FOUND : - kind == PURE_FAULT_EXISTS ? PURE_CONFLICT : PURE_INVALID; -} - -static bool reserve(PureArena *arena, size_t more) { - if (more <= arena->capacity - arena->used) return true; - size_t capacity = arena->capacity ? arena->capacity : 256; - while (capacity - arena->used < more) { - if (capacity > SIZE_MAX / 2) return false; - capacity *= 2; - } - unsigned char *bytes = realloc(arena->bytes, capacity); - if (!bytes) return false; - arena->bytes = bytes; - arena->capacity = capacity; - return true; -} - -/* Visit a value's bytes in order, from wherever they live. */ -static PureResult visit_value(const PureApp *app, const PureArena *arena, PureBytes v, - bool (*visit)(void *, PureSpan), void *context) { - if (!v.length) return PURE_OK; - const unsigned char *direct = NULL; - switch (v.source) { - case FROM_PAYLOAD: direct = app->payload.data + v.offset; break; - case FROM_COPY: direct = app->copied + v.offset; break; - case FROM_CONSTANT: direct = app->module->constant_data + v.offset; break; - case FROM_ARENA: direct = arena->bytes + v.offset; break; - case FROM_TEXT: - return pure_draft_read_version(app->texts[v.owner], v.version, (size_t)v.offset, (size_t)v.length, - visit, context); - default: - return app->data.read(app->data.context, app->document, v.version, (size_t)v.offset, - (size_t)v.length, visit, context); - } - visit(context, (PureSpan){direct, (size_t)v.length}); - return PURE_OK; -} - -static bool append_span(void *context, PureSpan span) { - PureArena *arena = context; - memmove(arena->bytes + arena->used, span.data, span.size); - arena->used += span.size; - return true; -} - -/* Copy a value to the end of the arena. Room is reserved first, so reading - * an arena value while appending never follows a moved buffer. */ -static PureResult append_value(const PureApp *app, PureArena *arena, PureBytes v) { - if (!reserve(arena, (size_t)v.length)) return PURE_ALLOCATION; - return visit_value(app, arena, v, append_span, arena); -} - -/* A contiguous span of a value, copying into the arena when it is spread out. */ -static PureResult contiguous(const PureApp *app, PureArena *arena, PureBytes v, PureSpan *out) { - if (v.source == FROM_PAYLOAD || v.source == FROM_COPY || v.source == FROM_CONSTANT) { - const unsigned char *base = v.source == FROM_PAYLOAD ? app->payload.data : - v.source == FROM_COPY ? app->copied : app->module->constant_data; - *out = (PureSpan){v.length ? base + v.offset : NULL, (size_t)v.length}; - return PURE_OK; - } - if (v.source == FROM_ARENA) { - *out = (PureSpan){arena->bytes + v.offset, (size_t)v.length}; - return PURE_OK; - } - size_t start = arena->used; - PureResult result = append_value(app, arena, v); - *out = (PureSpan){arena->bytes + start, (size_t)v.length}; - return result; -} - -typedef struct { size_t count; unsigned unit; unsigned utf8; bool valid; } Measure; - -static bool measure(void *context, PureSpan span) { - Measure *m = context; - if (m->unit == 2) m->valid = m->valid && pure_utf8_step(&m->utf8, span); - else m->count += m->unit ? pure_utf8_newlines(span) : pure_utf8_scalars(span); - return true; -} - -static PureBytes text_of(const PureApp *app, uint32_t state) { - PureVersion version = pure_draft_version(app->texts[state]); - return (PureBytes){FROM_TEXT, (uint16_t)state, version, 0, version.bytes}; -} - -static void move_cursors(const PureModule *m, double *values, uint32_t text, size_t at, - size_t removed, size_t inserted) { - for (uint32_t c = 0; c < m->states; ++c) { - if (m->state_type[c] != PURE_STATE_CURSOR || m->state_ref[c] != text) continue; - double position = values[c]; - if (removed) { - if (position >= at + removed) position -= (double)removed; - else if (position > at) position = (double)at; - } else if (position > at || (position == at && !(m->state_flags[c] & PURE_CURSOR_STAYS))) - position += (double)inserted; - values[c] = position; - } -} - -/* Indices follow their item across an insert or remove; an index whose item - * was removed names the next item (or the last, or 0 in an empty list). */ -static void move_indices(const PureModule *m, double *values, uint32_t list, size_t count, - size_t at, bool inserted) { - double last = count ? (double)(count - 1) : 0; - for (uint32_t i = 0; i < m->states; ++i) { - if (m->state_type[i] != PURE_STATE_INDEX || m->state_ref[i] != list) continue; - if (inserted) { - if (count > 1 && values[i] >= (double)at) ++values[i]; - } else { - if (values[i] > (double)at) --values[i]; - values[i] = minimum(values[i], last); - } - } -} - -static bool text_field(const PureModule *m, uint32_t list, uint32_t field) { - return m->record_field_text[m->record[m->state_ref[list]].first + field]; -} - -/* Append bytes to the arena as a value of it. */ -static PureResult keep_bytes(PureArena *arena, const void *bytes, size_t length, PureBytes *out) { - if (!reserve(arena, length)) return PURE_ALLOCATION; - if (length) memcpy(arena->bytes + arena->used, bytes, length); - *out = (PureBytes){FROM_ARENA, 0, {0, 0}, arena->used, length}; - arena->used += length; - return PURE_OK; -} - -/* Run a verified block. Types and stack bounds were proven at load. A change - * the data refuses (a size limit, a missing item) fails the event with its - * fault; other failures pass through. */ -typedef struct { unsigned char *at; } Copy; - -static bool copy_out(void *context, PureSpan span) { - Copy *copy = context; - if (span.size) memcpy(copy->at, span.data, span.size); - copy->at += span.size; - return true; -} - -/* An output's message from its field values, laid out as a message's - * payload (size bytes), numbered and kept with the event's changes. */ -static PureResult send(PureApp *app, uint32_t output, const Slot *fields, const PureArena *arena, size_t size) { - const PureModule *m = app->module; - const PureInputShape *shape = &m->input[output]; - if (app->sent_count == app->sent_capacity) { - size_t capacity = app->sent_capacity ? 2 * app->sent_capacity : 16; - struct PureSent *grown = realloc(app->sent, capacity * sizeof(*grown)); - if (!grown) return PURE_ALLOCATION; - app->sent = grown; - app->sent_capacity = capacity; - } - unsigned char *payload = malloc(size ? size : 1); - if (!payload) return PURE_ALLOCATION; - Copy copy = {payload + shape->prefix}; - for (uint32_t f = 0; f < shape->field_count; ++f) { - const PureField *field = &m->field[shape->field_start + f]; - if (field->type == PURE_FIELD_U32) write32(payload + field->offset, (uint32_t)fields[f].num); - else if (field->type == PURE_FIELD_NUMBER) { - uint64_t bits; - memcpy(&bits, &fields[f].num, 8); - write32(payload + field->offset, (uint32_t)bits); - write32(payload + field->offset + 4, (uint32_t)(bits >> 32)); - } else { - if (field->offset != PURE_FIELD_REST) write32(payload + field->offset, (uint32_t)fields[f].bytes.length); - PureResult result = visit_value(app, arena, fields[f].bytes, copy_out, ©); - if (result != PURE_OK) { free(payload); return result; } - } - } - app->sent[app->sent_count++] = (struct PureSent){output, ++app->last_sent[output], payload, size, false}; - app->links_changed = true; - return PURE_OK; -} - -static PureResult run(const PureModule *m, double *values, PureBlock block, Event *ev, Slot *result) { - Slot stack[PURE_MODULE_STACK]; - size_t sp = 0; - PureApp *app = ev->app; - PureArena *arena = ev->arena; - PureResult status; - PureFault kind; - const PureOp *op = NULL; - for (uint32_t pc = block.start, end = block.start + block.length; pc < end;) { - op = &m->code[pc++]; - uint32_t arg = op->arg; - Slot *top = sp ? &stack[sp - 1] : stack; - switch (op->op) { - case PURE_OP_CONST: stack[sp++].num = op->value; continue; - case PURE_OP_LOAD: stack[sp++].num = values[arg]; continue; - case PURE_OP_DT: stack[sp++].num = ev->dt; continue; - case PURE_OP_NOW: stack[sp++].num = app->now; continue; - case PURE_OP_FLOOR: top->num = floor(top->num); continue; - case PURE_OP_STORE: case PURE_OP_CUR_STORE: values[arg] = stack[--sp].num; continue; - case PURE_OP_JZ: if (stack[--sp].num == 0) pc = arg; continue; - case PURE_OP_JMP: pc = arg; continue; - case PURE_OP_NEG: top->num = -top->num; continue; - case PURE_OP_NOT: top->num = top->num == 0; continue; - case PURE_OP_CLAMP: { - double high = stack[--sp].num, low = stack[--sp].num; - if (low > high) { kind = PURE_FAULT_CLAMP; goto failed; } - stack[sp - 1].num = maximum(low, minimum(stack[sp - 1].num, high)); - continue; - } - case PURE_OP_FIELD: { - const PureField *field = &m->field[ev->shape->field_start + arg]; - if (field->type == PURE_FIELD_U32) stack[sp++].num = read32(app->payload.data + field->offset); - else if (field->type == PURE_FIELD_NUMBER) stack[sp++].num = read_double(app->payload.data + field->offset); - else stack[sp++].bytes = (PureBytes){FROM_PAYLOAD, 0, {0, 0}, ev->spans[arg][0], ev->spans[arg][1]}; - continue; - } - case PURE_OP_DOC: case PURE_OP_DOC_SIZE: { - if (op->op == PURE_OP_DOC && !ev->exists) { kind = PURE_FAULT_NOT_FOUND; goto failed; } - PureVersion version = app->data.version(app->data.context, app->document); - if (op->op == PURE_OP_DOC) stack[sp++].bytes = (PureBytes){FROM_DOCUMENT, 0, version, 0, version.bytes}; - else stack[sp++].num = (double)version.bytes; - continue; - } - case PURE_OP_LEN: case PURE_OP_TEXT_BYTES: top->num = (double)top->bytes.length; continue; - case PURE_OP_REPLACE: { - PureBytes insert = stack[--sp].bytes; - double last = stack[--sp].num, first = stack[--sp].num; - double size = (double)app->data.version(app->data.context, app->document).bytes; - if (first != floor(first) || last != floor(last) || first < 0 || last < first || last > size) { - kind = PURE_FAULT_LIMIT; - goto failed; - } - PureSpan span; - size_t mark = arena->used; - if ((status = contiguous(app, arena, insert, &span)) == PURE_OK) - status = app->data.splice(app->data.context, app->document, (size_t)first, (size_t)(last - first), span); - arena->used = mark; - if (status != PURE_OK) goto refused; - ev->modified = ev->exists = true; - continue; - } - /* A reply outlives the arena mark of later statements: keep it by value. */ - case PURE_OP_REPLY: app->reply = stack[--sp].bytes; app->has_reply = true; continue; - case PURE_OP_REQUIRE: if (stack[--sp].num == 0) { kind = PURE_FAULT_LIMIT; goto failed; } continue; - case PURE_OP_TEXT_CONST: - stack[sp++].bytes = (PureBytes){FROM_CONSTANT, 0, {0, 0}, m->constant_offset[arg], m->constant_length[arg]}; - continue; - case PURE_OP_TEXT_LOAD: stack[sp++].bytes = text_of(app, arg); continue; - case PURE_OP_TEXT_INPUT: stack[sp++].bytes = (PureBytes){FROM_PAYLOAD, 0, {0, 0}, 0, app->payload.size}; continue; - case PURE_OP_NUM_TEXT: { - char digits[32]; - if ((status = keep_bytes(arena, digits, pure_format_number(top->num, digits), &top->bytes)) != PURE_OK) - return status; - continue; - } - case PURE_OP_CONCAT: { - PureBytes right = stack[--sp].bytes, left = stack[sp - 1].bytes; - size_t start = arena->used; - if ((status = append_value(app, arena, left)) != PURE_OK || - (status = append_value(app, arena, right)) != PURE_OK) return status; - stack[sp - 1].bytes = (PureBytes){FROM_ARENA, 0, {0, 0}, start, left.length + right.length}; - continue; - } - case PURE_OP_CHARS: case PURE_OP_LINES: case PURE_OP_TEXT_FROM: { - Measure count = {0, op->op == PURE_OP_LINES ? 1 : op->op == PURE_OP_TEXT_FROM ? 2 : 0, 0, true}; - if ((status = visit_value(app, arena, top->bytes, measure, &count)) != PURE_OK) return status; - if (op->op != PURE_OP_TEXT_FROM) top->num = (double)(count.count + (op->op == PURE_OP_LINES)); - else if (!count.valid || !pure_utf8_complete(count.utf8)) { kind = PURE_FAULT_ENCODING; goto failed; } - continue; - } - /* Positions: a cursor or an index, with the text or list it is in. */ - case PURE_OP_CUR_LOAD: case PURE_OP_INDEX_LOAD: - stack[sp].num = values[arg]; - stack[sp++].bytes.owner = m->state_ref[arg]; - continue; - case PURE_OP_CUR_START: case PURE_OP_CUR_END: case PURE_OP_INDEX_FIRST: case PURE_OP_INDEX_LAST: - case PURE_OP_ROW: { - size_t last = op->op == PURE_OP_CUR_END ? pure_draft_size(app->texts[arg]) : - op->op == PURE_OP_INDEX_LAST ? pure_list_count(app->lists[arg]) : 0; - stack[sp].num = op->op == PURE_OP_ROW ? (double)ev->row : - op->op == PURE_OP_INDEX_LAST && last ? (double)(last - 1) : (double)last; - stack[sp++].bytes.owner = (uint16_t)arg; - continue; - } - case PURE_OP_CUR_MOVE: case PURE_OP_INDEX_MOVE: { - double count = stack[--sp].num; - Slot *at = &stack[sp - 1]; - if (count != floor(count)) { kind = PURE_FAULT_LIMIT; goto failed; } - if (op->op == PURE_OP_INDEX_MOVE) { - size_t items = pure_list_count(app->lists[at->bytes.owner]); - at->num = maximum(0, minimum(at->num + count, items ? (double)(items - 1) : 0)); - continue; - } - PureDraft *text = app->texts[at->bytes.owner]; - PureVersion version = pure_draft_version(text); - at->num = (double)(arg == PURE_UNIT_CHAR ? pure_text_step(text, version, (size_t)at->num, count) - : pure_text_line_step(text, version, (size_t)at->num, count)); - continue; - } - case PURE_OP_CUR_COUNT: { - PureDraft *text = app->texts[top->bytes.owner]; - top->num = (double)pure_text_count(text, pure_draft_version(text), (size_t)top->num, arg); - continue; - } - case PURE_OP_TEXT_INSERT: case PURE_OP_TEXT_SET: { - PureBytes insert = stack[--sp].bytes; - size_t at = op->op == PURE_OP_TEXT_INSERT ? (size_t)stack[--sp].num : 0; - size_t removed = op->op == PURE_OP_TEXT_SET ? pure_draft_size(app->texts[arg]) : 0; - PureSpan span; - size_t mark = arena->used; - if ((status = contiguous(app, arena, insert, &span)) == PURE_OK) - status = pure_draft_splice(app->texts[arg], at, removed, span); - arena->used = mark; - if (status != PURE_OK) goto refused; - if (op->op == PURE_OP_TEXT_SET) { - for (uint32_t c = 0; c < m->states; ++c) - if (m->state_type[c] == PURE_STATE_CURSOR && m->state_ref[c] == arg) values[c] = 0; - } else move_cursors(m, values, arg, at, 0, span.size); - continue; - } - case PURE_OP_TEXT_REMOVE: { - double y = stack[--sp].num, x = stack[--sp].num; - size_t first = (size_t)minimum(x, y), last = (size_t)maximum(x, y); - if ((status = pure_draft_splice(app->texts[arg], first, last - first, (PureSpan){0})) != PURE_OK) goto refused; - if (last > first) move_cursors(m, values, arg, first, last - first, 0); - continue; - } - case PURE_OP_TEXT_LINE: { - PureDraft *text = app->texts[arg]; - PureVersion version = pure_draft_version(text); - size_t first = pure_text_line_start(text, version, (size_t)top->num); - size_t last = pure_text_line_end(text, version, (size_t)top->num); - top->bytes = (PureBytes){FROM_TEXT, (uint16_t)arg, version, first, last - first}; - continue; - } - case PURE_OP_BYTES_FROM: case PURE_OP_INDEX_NUM: continue; /* the same bytes or position, retyped */ - case PURE_OP_LIST_COUNT: stack[sp++].num = (double)pure_list_count(app->lists[arg]); continue; - case PURE_OP_SEND: { - const PureInputShape *output = &m->input[arg]; - size_t base = sp - output->field_count, size = output->prefix; - sp = base; - for (uint32_t f = 0; f < output->field_count; ++f) { - uint8_t type = m->field[output->field_start + f].type; - double number = stack[base + f].num; - if (type == PURE_FIELD_BYTES || type == PURE_FIELD_TEXT) size += (size_t)stack[base + f].bytes.length; - else if (type == PURE_FIELD_U32 && !(number >= 0 && number <= UINT32_MAX && number == floor(number))) { - kind = PURE_FAULT_LIMIT; - goto failed; - } - } - if (size > UINT32_MAX) { kind = PURE_FAULT_LIMIT; goto failed; } - if ((status = send(app, arg, stack + base, arena, size)) != PURE_OK) return status; - continue; - } - case PURE_OP_LIST_APPEND: case PURE_OP_LIST_INSERT: { - PureList *list = app->lists[arg]; - const PureRecord *record = &m->record[m->state_ref[arg]]; - size_t base = sp - record->count, count = pure_list_count(list), at = count; - sp = base; - if (op->op == PURE_OP_LIST_INSERT && (at = (size_t)stack[--sp].num) > count) { - kind = PURE_FAULT_NOT_FOUND; - goto failed; - } - /* Room for every text first, so no span moves while the next is copied. */ - size_t mark = arena->used, total = 0; - for (unsigned f = 0; f < record->count; ++f) - if (text_field(m, arg, f)) total += (size_t)stack[base + f].bytes.length; - if (!reserve(arena, total)) return PURE_ALLOCATION; - PureCell cells[PURE_RECORD_FIELDS]; - for (unsigned f = 0; f < record->count; ++f) { - PureSpan span = {0}; - if (text_field(m, arg, f) && (status = contiguous(app, arena, stack[base + f].bytes, &span)) != PURE_OK) - return status; - cells[f] = (PureCell){stack[base + f].num, (unsigned char *)span.data, span.size}; - } - if (m->state_key[arg]) { /* a keyed list places the item by its key */ - bool found; - unsigned key = m->state_key[arg] - 1u; - at = pure_list_seek(list, key, cells[key], &found); - if (found) { arena->used = mark; kind = PURE_FAULT_EXISTS; goto failed; } - } - status = pure_list_insert(list, at, cells, &app->undo); - arena->used = mark; - if (status != PURE_OK) goto refused; - move_indices(m, values, arg, pure_list_count(list), at, true); - continue; - } - case PURE_OP_LIST_REMOVE: { - size_t at = (size_t)stack[--sp].num; - if ((status = pure_list_remove(app->lists[arg], at, &app->undo)) != PURE_OK) goto refused; - move_indices(m, values, arg, pure_list_count(app->lists[arg]), at, false); - continue; - } - case PURE_OP_ITEM_GET: { - uint32_t list = list_of(arg), field = item_field(arg); - size_t at = (size_t)top->num; - if (at >= pure_list_count(app->lists[list])) { kind = PURE_FAULT_NOT_FOUND; goto failed; } - PureCell cell = pure_list_get(app->lists[list], at, field); - /* A text is copied: the item may change or go before the value is used. */ - if (!text_field(m, list, field)) top->num = cell.num; - else if ((status = keep_bytes(arena, cell.text, cell.size, &top->bytes)) != PURE_OK) return status; - continue; - } - case PURE_OP_ITEM_SET: { - uint32_t list = list_of(arg), field = item_field(arg); - Slot value = stack[--sp]; - size_t at = (size_t)stack[--sp].num, mark = arena->used; - PureSpan span = {0}; - status = text_field(m, list, field) ? contiguous(app, arena, value.bytes, &span) : PURE_OK; - if (status == PURE_OK) - status = pure_list_set(app->lists[list], at, field, - (PureCell){value.num, (unsigned char *)span.data, span.size}, &app->undo); - arena->used = mark; - if (status != PURE_OK) goto refused; - continue; - } - case PURE_OP_INDEX_STORE: { - /* An index from before a remove may be past the end. */ - double position = stack[--sp].num; - if (position && position >= (double)pure_list_count(app->lists[m->state_ref[arg]])) { - kind = PURE_FAULT_NOT_FOUND; - goto failed; - } - values[arg] = position; - continue; - } - case PURE_OP_FOLD: { - /* The block that follows runs once per item, as a column of the list. */ - unsigned how = arg & 0xFF; - PureBlock inner = {pc, arg >> 16}; - PureList *list = app->lists[arg >> 8 & 0xFF]; - pc += inner.length; - double total = 0; - bool any = false; - Event per = *ev; - for (size_t row = 0, items = pure_list_count(list); row < items; ++row) { - Slot out[2]; - size_t mark = arena->used; - per.row = row; - status = run(m, values, inner, &per, out); - arena->used = mark; - if (status != PURE_OK) return status; - double value = out[0].num; - if (how == PURE_FOLD_COUNT || how == PURE_FOLD_FIRST ? value == 0 : out[1].num == 0) continue; - if (how == PURE_FOLD_FIRST) { total = (double)row; any = true; break; } - total = how == PURE_FOLD_COUNT ? total + 1 : how == PURE_FOLD_SUM ? total + value : - !any ? value : how == PURE_FOLD_MAX ? maximum(total, value) : minimum(total, value); - any = true; - } - if (!any && how >= PURE_FOLD_MAX) { kind = PURE_FAULT_NOT_FOUND; goto failed; } - if (!isfinite(total)) { kind = PURE_FAULT_NON_FINITE; goto failed; } - stack[sp].num = total; - stack[sp++].bytes.owner = (uint16_t)(arg >> 8 & 0xFF); /* first: the list the index is in */ - continue; - } - case PURE_OP_TEXT_COMPARE: { - /* Texts order by their bytes: a shorter text before a longer one it begins. */ - PureBytes right = stack[--sp].bytes, left = stack[sp - 1].bytes; - size_t mark = arena->used; - PureSpan a, b; - if (!reserve(arena, (size_t)(left.length + right.length))) return PURE_ALLOCATION; - if ((status = contiguous(app, arena, left, &a)) != PURE_OK || - (status = contiguous(app, arena, right, &b)) != PURE_OK) return status; - size_t shorter = a.size < b.size ? a.size : b.size; - int order = shorter ? memcmp(a.data, b.data, shorter) : 0; - if (!order) order = (a.size > b.size) - (a.size < b.size); - arena->used = mark; - stack[sp - 1].num = arg == PURE_LESS ? order < 0 : arg == PURE_LESS_EQUAL ? order <= 0 : - arg == PURE_GREATER ? order > 0 : arg == PURE_GREATER_EQUAL ? order >= 0 : - arg == PURE_EQUAL ? order == 0 : order != 0; - continue; - } - case PURE_OP_LIST_FIND: case PURE_OP_LIST_HAS: { - /* A keyed list's item by key, found by halving: FIND gives its index, HAS whether it is there. */ - unsigned key = m->state_key[arg] - 1u; - size_t mark = arena->used; - PureCell cell = {top->num, NULL, 0}; - if (text_field(m, arg, key)) { - PureSpan span; - if ((status = contiguous(app, arena, top->bytes, &span)) != PURE_OK) return status; - cell = (PureCell){0, (unsigned char *)span.data, span.size}; - } - bool found; - size_t at = pure_list_seek(app->lists[arg], key, cell, &found); - arena->used = mark; - if (op->op == PURE_OP_LIST_HAS) { top->num = found; continue; } - if (!found) { kind = PURE_FAULT_NOT_FOUND; goto failed; } - top->num = (double)at; - top->bytes.owner = (uint16_t)arg; - continue; - } - case PURE_OP_PICK: stack[sp] = stack[sp - 1 - arg]; ++sp; continue; - case PURE_OP_SLIDE: stack[sp - 1 - arg] = stack[sp - 1]; sp -= arg; continue; - default: break; - } - double b = stack[--sp].num, a = stack[sp - 1].num, value; - switch (op->op) { - case PURE_OP_ADD: value = a + b; break; - case PURE_OP_SUB: value = a - b; break; - case PURE_OP_MUL: value = a * b; break; - case PURE_OP_DIV: - if (b == 0) { kind = PURE_FAULT_DIVIDE; goto failed; } - value = a / b; - break; - case PURE_OP_LT: value = a < b; break; - case PURE_OP_LE: value = a <= b; break; - case PURE_OP_GT: value = a > b; break; - case PURE_OP_GE: value = a >= b; break; - case PURE_OP_EQ: value = a == b; break; - case PURE_OP_NE: value = a != b; break; - case PURE_OP_AND: value = a != 0 && b != 0; break; - case PURE_OP_OR: value = a != 0 || b != 0; break; - case PURE_OP_MIN: value = minimum(a, b); break; - case PURE_OP_MOD: /* the divisor's sign, as floor division leaves it (a zero too) */ - if (b == 0) { kind = PURE_FAULT_DIVIDE; goto failed; } - value = fmod(a, b); - if (value != 0 && (value < 0) != (b < 0)) value += b; - else if (value == 0) value = copysign(0.0, b); - break; - default: value = maximum(a, b); break; - } - if (!isfinite(value)) { kind = PURE_FAULT_NON_FINITE; goto failed; } - stack[sp - 1].num = value; - } - if (result) memcpy(result, stack, sp * sizeof(*stack)); /* a view's value, or a fold block's numbers */ - return PURE_OK; -refused: - kind = status == PURE_LIMIT ? PURE_FAULT_LIMIT : status == PURE_NOT_FOUND ? PURE_FAULT_NOT_FOUND : PURE_FAULT_NONE; - if (kind == PURE_FAULT_NONE) return status; -failed: - return fault(app, kind, (uint32_t)(op - m->code)); -} - -PURE_COLD static PureResult new_text(PureDraft **out, const PureModule *m, uint32_t constant) { - *out = pure_draft_new(); - if (!*out) return PURE_ALLOCATION; - PureSpan initial = {m->constant_data + m->constant_offset[constant], m->constant_length[constant]}; - PureResult result = pure_draft_splice(*out, 0, 0, initial); - if (result != PURE_OK) { pure_draft_close(*out); *out = NULL; } - return result; -} - -PURE_COLD static PureList *new_list(const PureModule *m, uint32_t record) { - return pure_list_new(m->record[record].count, m->record_field_text + m->record[record].first); -} - -static bool pending(const PureApp *app, const void *document) { - for (unsigned i = 0; i < app->pending_count; ++i) if (app->pending[i].document == document) return true; - return false; -} - -void pure_app_finish(PureApp *app) { - /* A document in the unsaved burst stays open until the save. */ - if (app->document && !pending(app, app->document)) app->data.close(app->data.context, app->document); - free(app->copied); - app->document = NULL; - app->copied = NULL; - app->has_reply = false; - app->payload = (PureSpan){0}; - app->arena.used = 0; -} - -/* The start of a burst: what a failed save returns to. Text and stored list - * drafts stay held until the save, so every version since stays readable. */ -static void open_burst(PureApp *app) { - if (app->open) return; - const PureModule *m = app->module; - memcpy(app->durable_values, app->values, sizeof(app->values)); - app->durable_ticks = app->ticks; - app->durable_events = app->events; - for (uint32_t i = 0; i < m->states; ++i) { - if (app->texts[i]) { - pure_draft_hold(app->texts[i], true); - app->durable_texts[i] = pure_draft_version(app->texts[i]); - } - if (app->list_drafts[i]) { - pure_draft_hold(app->list_drafts[i], true); - app->durable_lists[i] = pure_draft_version(app->list_drafts[i]); - } - } - app->open = true; -} - -/* End a burst, keeping it (saved) or undoing all of it. The documents it held - * close, except the current event's, which pure_app_finish closes. */ -static void close_burst(PureApp *app, bool keep) { - for (unsigned i = 0; i < app->pending_count; ++i) - if (app->pending[i].document != app->document) app->data.close(app->data.context, app->pending[i].document); - app->pending_count = 0; - if (!app->open) return; - const PureModule *m = app->module; - for (uint32_t i = 0; i < m->states; ++i) { - if (app->texts[i]) { - if (!keep) pure_draft_restore(app->texts[i], app->durable_texts[i]); - pure_draft_hold(app->texts[i], false); - } - if (app->list_drafts[i]) { - if (!keep) pure_draft_restore(app->list_drafts[i], app->durable_lists[i]); - pure_draft_hold(app->list_drafts[i], false); - } - } - if (keep) pure_undo_commit(&app->undo); - else { - pure_undo_rollback(&app->undo, 0); - memcpy(app->values, app->durable_values, sizeof(app->values)); - app->ticks = app->durable_ticks; - app->events = app->durable_events; - ++app->changes; - } - app->open = false; -} - -static void drop_sent(PureApp *app, size_t count) { - while (app->sent_count > count) free(app->sent[--app->sent_count].payload); -} - -/* After a save: what it stored is the links' saved state (trimmed sends go), - * or, when it failed, the links return to the last saved state. */ -static void settle_links(PureApp *app, bool saved) { - if (saved) { - size_t kept = 0; - for (size_t i = 0; i < app->sent_count; ++i) { - if (app->sent[i].trimmed) free(app->sent[i].payload); - else app->sent[kept++] = app->sent[i]; - } - app->sent_count = app->sent_durable = kept; - memcpy(app->saved_sent, app->last_sent, sizeof(app->saved_sent)); - for (unsigned i = 0; i < app->heard_count; ++i) app->heard[i].saved = app->heard[i].seq; - } else { - drop_sent(app, app->sent_durable); - for (size_t i = 0; i < app->sent_count; ++i) app->sent[i].trimmed = false; - memcpy(app->last_sent, app->saved_sent, sizeof(app->last_sent)); - unsigned kept = 0; - for (unsigned i = 0; i < app->heard_count; ++i) - if (app->heard[i].saved) { - app->heard[kept] = app->heard[i]; - app->heard[kept++].seq = app->heard[i].saved; - } - app->heard_count = kept; - } - app->links_changed = false; -} - -PureResult pure_app_save(PureApp *app) { - void *documents[PURE_APP_PENDING]; - int64_t revisions[PURE_APP_PENDING]; - size_t count = 0; - for (unsigned i = 0; i < app->pending_count; ++i) - if (app->pending[i].edited) documents[count++] = app->pending[i].document; - PureResult result = pure_save_commit(app, app->values, documents, count, revisions); - settle_links(app, result == PURE_OK); - close_burst(app, result == PURE_OK); - return result; -} - -bool pure_app_defer(PureApp *app, bool deferred) { - if (deferred && !pure_save_enabled(app)) return false; - app->deferred = deferred; - return true; -} - -/* ---- Snapshots. Snapshots of one app form a chain from oldest to newest, - * each holding the next, so they go oldest first. List items the app lets go - * while a snapshot is its newest may be read by that one and older ones, so - * they go with it. */ -struct PureSnapshot { - unsigned refs; - const PureModule *module; - uint64_t changes; - double values[PURE_MODULE_STATES]; - PureDraft *texts[PURE_MODULE_STATES]; - PureList *lists[PURE_MODULE_STATES]; - PureRetired retired; - struct PureSnapshot *newer; -}; - -void pure_snapshot_hold(PureSnapshot *snapshot) { __atomic_add_fetch(&snapshot->refs, 1, __ATOMIC_RELAXED); } - -void pure_snapshot_release(PureSnapshot *snapshot) { - while (snapshot && !__atomic_sub_fetch(&snapshot->refs, 1, __ATOMIC_ACQ_REL)) { - PureSnapshot *newer = snapshot->newer; - for (uint32_t i = 0; i < PURE_MODULE_STATES; ++i) { - pure_draft_close(snapshot->texts[i]); - pure_list_free(snapshot->lists[i]); - } - pure_retired_free(&snapshot->retired); - free(snapshot); - snapshot = newer; - } -} - -void pure_app_stop(PureApp *app) { - pure_app_finish(app); - close_burst(app, true); /* unsaved changes are dropped with the app */ - for (uint32_t i = 0; i < PURE_MODULE_STATES && !app->reading; ++i) { - pure_draft_close(app->texts[i]); - pure_list_free(app->lists[i]); - pure_draft_close(app->list_drafts[i]); - } - /* Items the newest snapshot may still read go when it goes. */ - if (app->snapshot) { - app->snapshot->retired = app->retired; - pure_snapshot_release(app->snapshot); - } else - pure_retired_free(&app->retired); - pure_draft_close(app->stored_values); - drop_sent(app, 0); - free(app->sent); - free(app->heard); - free(app->carried); - pure_draft_close(app->stored_links); - free(app->pending); - pure_undo_free(&app->undo); - for (unsigned i = 0; i < app->row_tables; ++i) free(app->rows[i].items); - free(app->rows); - free(app->arena.bytes); - free(app->view_arena.bytes); - memset(app, 0, sizeof(*app)); -} - -/* Where one event began inside the burst: a failed event returns there. */ -typedef struct { - PureVersion texts[PURE_MODULE_STATES], lists[PURE_MODULE_STATES]; - size_t undo, sent; - uint64_t last_sent[PURE_MODULE_INPUTS]; - bool opened; -} Mark; - -static void mark_event(PureApp *app, Mark *mark) { - mark->opened = !app->open; - open_burst(app); - for (uint32_t i = 0; i < app->module->states; ++i) { - if (app->texts[i]) mark->texts[i] = pure_draft_version(app->texts[i]); - if (app->list_drafts[i]) mark->lists[i] = pure_draft_version(app->list_drafts[i]); - } - mark->undo = app->undo.count; - mark->sent = app->sent_count; - memcpy(mark->last_sent, app->last_sent, sizeof(mark->last_sent)); -} - -static void undo_event(PureApp *app, const Mark *mark) { - for (uint32_t i = 0; i < app->module->states; ++i) { - if (app->texts[i]) pure_draft_restore(app->texts[i], mark->texts[i]); - if (app->list_drafts[i]) pure_draft_restore(app->list_drafts[i], mark->lists[i]); - } - pure_undo_rollback(&app->undo, mark->undo); - drop_sent(app, mark->sent); - memcpy(app->last_sent, mark->last_sent, sizeof(app->last_sent)); - if (mark->opened) close_burst(app, true); /* nothing happened: release the holds */ -} - -/* The last number heard from source, or NULL. */ -static struct PureHeard *heard_from(PureApp *app, const char *source) { - for (unsigned i = 0; i < app->heard_count; ++i) - if (!strcmp(app->heard[i].source, source)) return &app->heard[i]; - return NULL; -} - -/* Hear seq from source (room for a new source was made before). */ -static void hear(PureApp *app, const char *source, uint64_t seq) { - struct PureHeard *heard = heard_from(app, source); - if (!heard) { - heard = &app->heard[app->heard_count++]; - *heard = (struct PureHeard){.saved = 0}; - memcpy(heard->source, source, strlen(source) + 1); - } - heard->seq = seq; - app->links_changed = true; -} - -static PureResult keep_reply(PureApp *app); - -/* One event: every handler for it runs in module order against pending - * states, all or nothing. A key, text or message event saves before it - * returns unless saving is deferred; a frame leaves that to the next event - * (frames come 60 times a second). A message's document joins the burst. */ -static PureResult dispatch(PureApp *app, uint32_t event, Event *ev) { - const PureModule *m = app->module; - double pending[PURE_MODULE_STATES]; - Mark mark; - mark_event(app, &mark); - memcpy(pending, app->values, sizeof(app->values)); - app->fault = PURE_FAULT_NONE; - app->fault_pc = UINT32_MAX; - PureResult result = PURE_OK; - for (uint32_t i = 0; i < m->handlers && result == PURE_OK; ++i) - if (m->handler[i].event == event) result = run(m, pending, m->handler[i].code, ev, NULL); - /* An edit needs the revision it was made against. A save rebases an - * edited document and releasing texts may reclaim earlier text versions: - * keep such a reply by copying it first. */ - if (result == PURE_OK && ev->modified && ev->base < 0) result = PURE_INVALID; - if (result == PURE_OK && app->has_reply && - ((app->reply.source == FROM_DOCUMENT && ev->modified) || app->reply.source == FROM_TEXT)) - result = keep_reply(app); - if (result != PURE_OK) { - undo_event(app, &mark); - return result; - } - memcpy(app->values, pending, sizeof(pending)); - ++app->events; - ++app->changes; - if (!event) ++app->ticks; - if (app->document) { - app->pending[app->pending_count].document = app->document; - app->pending[app->pending_count].edited = ev->modified; - memcpy(app->pending[app->pending_count++].key, ev->key, strlen(ev->key) + 1); - app->revision += ev->modified; /* a save moves an edited document one revision */ - } - if (app->delivering.pending) { - hear(app, app->delivering.source, app->delivering.seq); - app->delivering.pending = false; - } - return event && !app->deferred ? pure_app_save(app) : PURE_OK; -} - -PureResult pure_app_frame(PureApp *app) { - pure_app_finish(app); - Event ev = {.app = app, .arena = &app->arena, .dt = 1.0 / 60.0}; - return dispatch(app, 0, &ev); -} - -PureResult pure_app_tick(PureApp *app, uint32_t input) { - if (input >= app->module->inputs || app->module->input[input].kind != PURE_KIND_TIMER) return PURE_INVALID; - pure_app_finish(app); - Event ev = {.app = app, .arena = &app->arena}; - return dispatch(app, 1 + 2 * input, &ev); -} - -PureResult pure_app_input(PureApp *app, uint32_t input, PureEdge edge) { - if (input >= app->module->inputs || app->module->input[input].kind != PURE_KIND_KEY || - (edge != PURE_EDGE_DOWN && edge != PURE_EDGE_UP)) return PURE_INVALID; - pure_app_finish(app); - Event ev = {.app = app, .arena = &app->arena}; - return dispatch(app, 1 + 2 * input + (edge == PURE_EDGE_UP), &ev); -} - -PureResult pure_app_point(PureApp *app, uint32_t state, size_t position) { - const PureModule *m = app->module; - pure_app_finish(app); - if (state >= m->states || m->state_type[state] != PURE_STATE_INDEX) return PURE_INVALID; - if (position >= pure_list_count(app->lists[m->state_ref[state]])) return PURE_NOT_FOUND; - open_burst(app); /* what a failed save returns to */ - app->fault = PURE_FAULT_NONE; - app->fault_pc = UINT32_MAX; - app->values[state] = (double)position; - ++app->events; - ++app->changes; - return app->deferred ? PURE_OK : pure_app_save(app); -} - -PureResult pure_app_text(PureApp *app, uint32_t input, PureSpan text) { - pure_app_finish(app); - if (input >= app->module->inputs || app->module->input[input].kind != PURE_KIND_TEXT || - (text.size && !text.data) || !pure_utf8_valid(text)) return PURE_INVALID; - app->payload = text; - Event ev = {.app = app, .arena = &app->arena}; - PureResult result = dispatch(app, 1 + 2 * input, &ev); - app->payload = (PureSpan){0}; - return result; -} - -static PureResult keep_reply(PureApp *app) { - size_t length = (size_t)app->reply.length, start = app->arena.used; - unsigned char *copy = malloc(length ? length : 1); - if (!copy) return PURE_ALLOCATION; - PureResult result = append_value(app, &app->arena, app->reply); - if (result != PURE_OK) { free(copy); return result; } - if (length) memcpy(copy, app->arena.bytes + start, length); - app->arena.used = start; - app->copied = copy; - app->reply = (PureBytes){FROM_COPY, 0, {0, 0}, 0, length}; - return PURE_OK; -} - -/* A message whose variable fields are where placed says (start and size in - * the payload), or, without it, one after another after the prefix. */ -static PureResult message(PureApp *app, uint32_t input, const char *key, int64_t base, PureSpan payload, - const uint32_t *placed) { - pure_app_finish(app); - const PureModule *m = app->module; - app->fault = PURE_FAULT_NONE; - app->fault_pc = UINT32_MAX; - app->revision = 0; - if (input >= m->inputs || m->input[input].kind != PURE_KIND_MESSAGE || base < -1 || - (payload.size && !payload.data) || (m->input[input].keyed ? !document_key(key) : key != NULL)) - return PURE_INVALID; - const PureInputShape *shape = &m->input[input]; - if (payload.size < shape->prefix) return fault(app, PURE_FAULT_LIMIT, UINT32_MAX); - /* Where each variable field is; a length past the payload, a text that - * is not UTF-8 or a number that is not finite fails the message. */ - uint32_t spans[PURE_MODULE_FIELDS][2]; - size_t at = shape->prefix; - for (uint32_t f = 0; f < shape->field_count; ++f) { - const PureField *field = &m->field[shape->field_start + f]; - if (field->type == PURE_FIELD_NUMBER && !isfinite(read_double(payload.data + field->offset))) - return fault(app, PURE_FAULT_NON_FINITE, UINT32_MAX); - if (field->type != PURE_FIELD_BYTES && field->type != PURE_FIELD_TEXT) continue; - size_t start = placed ? placed[2 * f] : at; - if (start > payload.size) return fault(app, PURE_FAULT_LIMIT, UINT32_MAX); - size_t size = placed ? placed[2 * f + 1] : field->offset == PURE_FIELD_REST ? payload.size - at : - read32(payload.data + field->offset); - if (size > payload.size - start) return fault(app, PURE_FAULT_LIMIT, UINT32_MAX); - if (field->type == PURE_FIELD_TEXT && !pure_utf8_valid((PureSpan){size ? payload.data + start : NULL, size})) - return fault(app, PURE_FAULT_ENCODING, UINT32_MAX); - spans[f][0] = (uint32_t)start; - spans[f][1] = (uint32_t)size; - at = start + size; - } - if (shape->keyed && m->collections) { - if (!app->data.open) return PURE_INVALID; - /* A key joins a burst once, so its revision moves once per save. */ - bool seen = app->pending_count == PURE_APP_PENDING; - for (unsigned i = 0; i < app->pending_count && !seen; ++i) seen = !strcmp(app->pending[i].key, key); - if (seen) { - PureResult saved = pure_app_save(app); - if (saved != PURE_OK) return saved; - } - if (app->pending_count == app->pending_capacity) { - unsigned capacity = app->pending_capacity ? 2 * app->pending_capacity : 16; - struct PurePending *grown = realloc(app->pending, capacity * sizeof(*grown)); - if (!grown) return PURE_ALLOCATION; - app->pending = grown; - app->pending_capacity = capacity; - } - PureResult opened = app->data.open(app->data.context, 0, key, &app->document, &app->revision); - if (opened != PURE_OK) { app->document = NULL; return opened; } - if (base >= 0 && base != app->revision) { pure_app_finish(app); return PURE_CONFLICT; } - } - app->payload = payload; - Event ev = {.app = app, .shape = shape, .key = key, .base = base, .arena = &app->arena, .exists = app->revision > 0, - .spans = spans}; - PureResult result = dispatch(app, 1 + 2 * input, &ev); - if (result != PURE_OK) pure_app_finish(app); - return result; -} - -PureResult pure_app_message(PureApp *app, uint32_t input, const char *key, int64_t base, PureSpan payload) { - return message(app, input, key, base, payload, NULL); -} - -PureResult pure_app_message_fields(PureApp *app, uint32_t input, const char *key, int64_t base, PureSpan payload, - const uint32_t *placed) { - return message(app, input, key, base, payload, placed); -} - -void pure_app_outbox(const PureApp *app, PureSentVisit visit, void *context) { - for (size_t i = 0; i < app->sent_durable; ++i) { - const struct PureSent *sent = &app->sent[i]; - if (!sent->trimmed && !visit(context, sent->output, sent->seq, (PureSpan){sent->payload, sent->size})) return; - } -} - -void pure_app_outbox_newest(const PureApp *app, PureSentVisit visit, void *context) { - for (size_t i = app->sent_durable; i-- > 0;) { - const struct PureSent *sent = &app->sent[i]; - if (!sent->trimmed && !visit(context, sent->output, sent->seq, (PureSpan){sent->payload, sent->size})) return; - } -} - -void pure_app_trim(PureApp *app, uint32_t output, uint64_t through) { - for (size_t i = 0; i < app->sent_durable; ++i) - if (app->sent[i].output == output && app->sent[i].seq <= through && !app->sent[i].trimmed) { - app->sent[i].trimmed = true; - app->links_changed = true; - } -} - -PureResult pure_app_deliver(PureApp *app, uint32_t input, const char *source, uint64_t seq, PureSpan payload, - const uint32_t *placed) { - const PureModule *m = app->module; - size_t length = source ? strlen(source) : 0; - app->duplicate = false; - if (input >= m->inputs || m->input[input].kind != PURE_KIND_MESSAGE || m->input[input].keyed || !length || - length > PURE_SOURCE || !seq) return PURE_INVALID; - for (size_t i = 0; i < length; ++i) - if (source[i] <= ' ' || source[i] > '~') return PURE_INVALID; - struct PureHeard *heard = heard_from(app, source); - if (heard && seq <= heard->seq) { - pure_app_finish(app); - app->duplicate = true; - return PURE_OK; - } - if (!heard && app->heard_count == app->heard_capacity) { - unsigned capacity = app->heard_capacity ? 2 * app->heard_capacity : 4; - struct PureHeard *grown = realloc(app->heard, capacity * sizeof(*grown)); - if (!grown) return PURE_ALLOCATION; - app->heard = grown; - app->heard_capacity = capacity; - } - app->delivering.source = source; - app->delivering.seq = seq; - app->delivering.pending = true; - PureResult result = message(app, input, NULL, -1, payload, placed); - if (!app->delivering.pending) return result; - /* Not applied. Refused by the app (a fault), the message was delivered - * all the same: its number is heard, saved like a key event's changes. */ - app->delivering.pending = false; - if (app->fault == PURE_FAULT_NONE) return result; - Mark mark; - mark_event(app, &mark); - hear(app, source, seq); - PureResult saved = app->deferred ? PURE_OK : pure_app_save(app); - return saved == PURE_OK ? result : saved; -} - -PureResult pure_app_reply(const PureApp *app, bool (*visit)(void *, PureSpan), void *context) { - if (!app->has_reply) return PURE_OK; - return visit_value(app, &app->arena, app->reply, visit, context); -} - -/* A view that fails reports it by its result; the fault stays the last event's. */ -static PureResult evaluate(PureApp *app, uint32_t view, size_t row, Slot *out) { - double values[PURE_MODULE_STATES]; - memcpy(values, app->values, sizeof(values)); - app->view_arena.used = 0; - Event ev = {.app = app, .arena = &app->view_arena, .row = row}; - PureFault fault = app->fault; - uint32_t fault_pc = app->fault_pc; - PureResult result = run(app->module, values, app->module->view_code[view], &ev, out); - app->fault = fault; - app->fault_pc = fault_pc; - return result; -} - -/* A filtered table's items, as fresh as the app's states: the filter runs - * once per item after each change (an item it fails on is left out), not - * once per cell. NULL for a table without a filter, or without room. */ -static struct PureRows *filtered(PureApp *app, uint32_t table) { - const PureModule *m = app->module; - if (!m->view_code[table].length) return NULL; - struct PureRows *rows = NULL; - for (unsigned i = 0; i < app->row_tables && !rows; ++i) if (app->rows[i].table == table) rows = &app->rows[i]; - if (!rows) { - struct PureRows *grown = realloc(app->rows, (app->row_tables + 1) * sizeof(*grown)); - if (!grown) return NULL; - app->rows = grown; - rows = &app->rows[app->row_tables++]; - *rows = (struct PureRows){.table = table, .changes = UINT64_MAX}; - } - if (rows->changes == app->changes) return rows; - rows->count = 0; - for (size_t item = 0, items = pure_list_count(app->lists[m->view_list[table]]); item < items; ++item) { - Slot keep; - if (evaluate(app, table, item, &keep) != PURE_OK || keep.num == 0) continue; - if (rows->count == rows->capacity) { - size_t capacity = rows->capacity ? 2 * rows->capacity : 16; - size_t *grown = realloc(rows->items, capacity * sizeof(*grown)); - if (!grown) return NULL; - rows->items = grown; - rows->capacity = capacity; - } - rows->items[rows->count++] = item; - } - rows->changes = app->changes; - return rows; -} - -bool pure_app_row_item(PureApp *app, uint32_t table, size_t row, size_t *item) { - const PureModule *m = app->module; - if (table >= m->views || m->view_type[table] != PURE_VIEW_TABLE) return false; - if (!m->view_code[table].length) { - *item = row; - return row < pure_list_count(app->lists[m->view_list[table]]); - } - struct PureRows *rows = filtered(app, table); - if (!rows || row >= rows->count) return false; - *item = rows->items[row]; - return true; -} - -bool pure_app_item_row(PureApp *app, uint32_t table, size_t item, size_t *row) { - const PureModule *m = app->module; - if (table >= m->views || m->view_type[table] != PURE_VIEW_TABLE) return false; - if (!m->view_code[table].length) { - *row = item; - return item < pure_list_count(app->lists[m->view_list[table]]); - } - struct PureRows *rows = filtered(app, table); - size_t low = 0, high = rows ? rows->count : 0; /* the items are in order */ - while (low < high) { - size_t middle = low + (high - low) / 2; - if (rows->items[middle] < item) low = middle + 1; - else high = middle; - } - if (low == (rows ? rows->count : 0) || rows->items[low] != item) return false; - *row = low; - return true; -} - -/* A view outside a table (column false), or a column and a row of its table. */ -static PureResult view_of(PureApp *app, uint32_t view, bool column, size_t row, uint8_t type, Slot *slot) { - const PureModule *m = app->module; - if (view >= m->views || m->view_type[view] != type || (m->view_table[view] != 0) != column) return PURE_INVALID; - if (column && !pure_app_row_item(app, m->view_table[view] - 1u, row, &row)) return PURE_NOT_FOUND; - return evaluate(app, view, row, slot); -} - -bool pure_app_view(PureApp *app, uint32_t view, double *out) { - Slot slot; - if (view_of(app, view, false, 0, PURE_VIEW_NUM, &slot) != PURE_OK) return false; - *out = slot.num; - return true; -} - -PureResult pure_app_view_text(PureApp *app, uint32_t view, bool (*visit)(void *, PureSpan), void *context) { - Slot slot; - PureResult result = view_of(app, view, false, 0, PURE_VIEW_TEXT, &slot); - return result == PURE_OK ? visit_value(app, &app->view_arena, slot.bytes, visit, context) : result; -} - -size_t pure_app_rows(PureApp *app, uint32_t table) { - const PureModule *m = app->module; - if (table >= m->views || m->view_type[table] != PURE_VIEW_TABLE) return 0; - struct PureRows *rows = filtered(app, table); - return rows ? rows->count : m->view_code[table].length ? 0 : pure_list_count(app->lists[m->view_list[table]]); -} - -bool pure_app_cell(PureApp *app, uint32_t column, size_t row, double *out) { - Slot slot; - if (view_of(app, column, true, row, PURE_VIEW_NUM, &slot) != PURE_OK) return false; - *out = slot.num; - return true; -} - -PureResult pure_app_cell_text(PureApp *app, uint32_t column, size_t row, bool (*visit)(void *, PureSpan), void *context) { - Slot slot; - PureResult result = view_of(app, column, true, row, PURE_VIEW_TEXT, &slot); - return result == PURE_OK ? visit_value(app, &app->view_arena, slot.bytes, visit, context) : result; -} - -PureSnapshot *pure_app_snapshot(PureApp *app) { - if (app->open || app->reading) return NULL; - const PureModule *m = app->module; - PureSnapshot *made = calloc(1, sizeof(*made)), *last = app->snapshot; - if (!made) return NULL; - made->refs = 1; - made->module = m; - made->changes = app->changes; - memcpy(made->values, app->values, sizeof(made->values)); - bool ok = true; - for (uint32_t i = 0; i < m->states && ok; ++i) { - if (app->texts[i]) ok = (made->texts[i] = pure_draft_freeze(app->texts[i])) != NULL; - if (!ok || !app->lists[i]) continue; - if (last && last->lists[i] && pure_list_current(last->lists[i], app->lists[i])) { - pure_list_hold(last->lists[i]); - made->lists[i] = last->lists[i]; - } else - ok = (made->lists[i] = pure_list_share(app->lists[i], &app->retired)) != NULL; - } - if (!ok) { - pure_snapshot_release(made); - return NULL; - } - made->refs = last ? 3 : 2; /* the app's, the caller's and the last one's */ - if (last) { - last->retired = app->retired; - last->newer = made; - pure_snapshot_release(last); - } else - pure_retired_free(&app->retired); /* shared only by a snapshot never made */ - app->retired = (PureRetired){0}; - app->snapshot = made; - return made; -} - -void pure_app_read(PureApp *reader, const PureSnapshot *snapshot) { - reader->module = snapshot->module; - reader->reading = true; - reader->changes = snapshot->changes; /* what the row cache is fresh for */ - memcpy(reader->values, snapshot->values, sizeof(reader->values)); - memcpy(reader->texts, snapshot->texts, sizeof(reader->texts)); - memcpy(reader->lists, snapshot->lists, sizeof(reader->lists)); -} - -/* A list with record's fields: each takes the field of `from` with the same - * name and kind, or its default. */ -PURE_COLD PureResult pure_list_migrate(PureList **out, const PureList *from, const char *const *names, - const uint8_t *text, unsigned count, const PureModule *m, uint32_t record) { - const PureRecord *r = &m->record[record]; - int source[PURE_RECORD_FIELDS]; - for (unsigned f = 0; f < r->count; ++f) { - source[f] = -1; - for (unsigned k = 0; k < count; ++k) - if (!strcmp(m->record_field_names[r->first + f], names[k]) && m->record_field_text[r->first + f] == text[k]) - source[f] = (int)k; - } - if (!(*out = new_list(m, record))) return PURE_ALLOCATION; - PureResult result = PURE_OK; - for (size_t at = 0; result == PURE_OK && at < pure_list_count(from); ++at) { - PureCell cells[PURE_RECORD_FIELDS]; - for (unsigned f = 0; f < r->count; ++f) { - uint32_t field = r->first + f, constant = (uint32_t)m->record_field_default[field]; - cells[f] = source[f] >= 0 ? pure_list_get(from, at, (unsigned)source[f]) : - m->record_field_text[field] ? - (PureCell){0, (unsigned char *)m->constant_data + m->constant_offset[constant], - m->constant_length[constant]} : - (PureCell){m->record_field_default[field], NULL, 0}; - } - result = pure_list_insert(*out, at, cells, NULL); - } - if (result != PURE_OK) { pure_list_free(*out); *out = NULL; } - return result; -} - -/* A kept list keeps its items; with the same record it is the same list. */ -PURE_COLD static PureResult keep_list(PureList **out, PureList *from, const PureModule *old, uint32_t old_record, - const PureModule *m, uint32_t record) { - const PureRecord *r = &m->record[record], *o = &old->record[old_record]; - const char *names[PURE_RECORD_FIELDS]; - bool same = r->count == o->count; - for (unsigned k = 0; k < o->count; ++k) { - names[k] = old->record_field_names[o->first + k]; - same = same && !strcmp(names[k], m->record_field_names[r->first + k]) && - old->record_field_text[o->first + k] == m->record_field_text[r->first + k]; - } - if (same) { *out = from; return PURE_OK; } - return pure_list_migrate(out, from, names, old->record_field_text + o->first, o->count, m, record); -} - -/* Open a module in an app. States keep their values by name and kind (texts - * their content, cursors their position in it, lists their items, moved by - * field name when the record changed); every other state starts from its - * stored value, or its initial one. A list whose record changed is stored - * again, and the next event or save publishes the module's states; a state - * the module drops stays stored as it was. Starting an app opens its module - * over none. */ -/* Outputs of old that module has by name and with the same fields keep their - * sends and numbers; one it lacks or changed refuses the module while sends - * are left on it (CONFLICT), and otherwise keeps its number for later. */ -PURE_COLD static PureResult remap_outputs(PureApp *app, const PureModule *old, const PureModule *module, bool apply) { - int map[PURE_MODULE_INPUTS]; - for (uint32_t i = 0; i < old->inputs; ++i) { - map[i] = -1; - if (old->input[i].kind != PURE_KIND_OUTPUT) continue; - int to = pure_module_input(module, old->input_names[i]); - bool same = to >= 0 && module->input[to].kind == PURE_KIND_OUTPUT && - module->input[to].field_count == old->input[i].field_count; - for (uint32_t f = 0; same && f < old->input[i].field_count; ++f) - same = module->field[module->input[to].field_start + f].type == old->field[old->input[i].field_start + f].type; - map[i] = same ? to : -1; - for (size_t s = 0; !same && s < app->sent_count; ++s) - if (app->sent[s].output == i) return PURE_CONFLICT; - } - if (!apply) return PURE_OK; - uint64_t last[PURE_MODULE_INPUTS] = {0}, saved[PURE_MODULE_INPUTS] = {0}; - for (uint32_t i = 0; i < old->inputs; ++i) { - if (old->input[i].kind != PURE_KIND_OUTPUT || !app->last_sent[i]) continue; - if (map[i] >= 0) { - last[map[i]] = app->last_sent[i]; - saved[map[i]] = app->saved_sent[i]; - continue; - } - struct PureCarried *grown = realloc(app->carried, (app->carried_count + 1) * sizeof(*grown)); - if (!grown) continue; /* its numbers would start again: kept only while memory allows */ - app->carried = grown; - memcpy(grown[app->carried_count].name, old->input_names[i], PURE_MODULE_NAME); - grown[app->carried_count++].last = app->last_sent[i]; - app->links_changed = true; - } - for (unsigned c = 0; c < app->carried_count;) { - int to = pure_module_input(module, app->carried[c].name); - if (to < 0 || module->input[to].kind != PURE_KIND_OUTPUT || last[to]) { ++c; continue; } - last[to] = saved[to] = app->carried[c].last; - app->carried[c] = app->carried[--app->carried_count]; - app->links_changed = true; - } - for (size_t s = 0; s < app->sent_count; ++s) app->sent[s].output = (uint32_t)map[app->sent[s].output]; - memcpy(app->last_sent, last, sizeof(last)); - memcpy(app->saved_sent, saved, sizeof(saved)); - return PURE_OK; -} - -PURE_COLD static PureResult open_module(PureApp *app, const PureModule *module) { - const PureModule *old = app->module; - bool stored = pure_save_enabled(app); - double values[PURE_MODULE_STATES] = {0}; - PureDraft *texts[PURE_MODULE_STATES] = {0}, *list_drafts[PURE_MODULE_STATES] = {0}; - PureList *lists[PURE_MODULE_STATES] = {0}; - PureVersion saved[PURE_MODULE_STATES] = {{0, 0}}, before[PURE_MODULE_STATES] = {{0, 0}}; - int kept[PURE_MODULE_STATES]; - bool rewrite[PURE_MODULE_STATES] = {false}; - PureResult result = PURE_OK; - for (uint32_t i = 0; i < module->states; ++i) { - int previous = old ? pure_module_state(old, module->state_names[i]) : -1; - kept[i] = previous >= 0 && old->state_type[previous] == module->state_type[i] ? previous : -1; - if (kept[i] >= 0) saved[i] = app->saved[kept[i]]; - if (module->state_type[i] == PURE_STATE_TEXT) { - if (kept[i] >= 0) texts[i] = app->texts[kept[i]]; - else if (result == PURE_OK) - result = stored ? pure_save_open_text(app, module, i, &texts[i], &saved[i]) : - new_text(&texts[i], module, module->state_ref[i]); - } else if (module->state_type[i] == PURE_STATE_LIST) { - if (result != PURE_OK) continue; - if (kept[i] >= 0) { - result = keep_list(&lists[i], app->lists[kept[i]], old, old->state_ref[kept[i]], module, - module->state_ref[i]); - list_drafts[i] = app->list_drafts[kept[i]]; - rewrite[i] = result == PURE_OK && list_drafts[i] && lists[i] != app->lists[kept[i]]; - } else if (stored) { - result = pure_save_open_list(app, module, i, &lists[i], &list_drafts[i], &saved[i]); - } else if (!(lists[i] = new_list(module, module->state_ref[i]))) result = PURE_ALLOCATION; - } else values[i] = kept[i] >= 0 ? app->values[kept[i]] : module->initial[i]; - } - /* A keyed list keeps its items in key order: one that was not keyed, or - * keyed by another field, is put in order and its indices follow their - * items. Two items with one key refuse the module: nothing is dropped. */ - size_t *moved[PURE_MODULE_STATES] = {0}; - for (uint32_t i = 0; i < module->states && result == PURE_OK; ++i) { - if (module->state_type[i] != PURE_STATE_LIST || !module->state_key[i]) continue; - size_t count = pure_list_count(lists[i]); - PureList *sorted = NULL; - if (count && !(moved[i] = malloc(count * sizeof(**moved)))) result = PURE_ALLOCATION; - else result = pure_list_sorted(lists[i], module->state_key[i] - 1u, &sorted, moved[i]); - if (!sorted) { - free(moved[i]); - moved[i] = NULL; - continue; - } - if (kept[i] < 0 || lists[i] != app->lists[kept[i]]) pure_list_free(lists[i]); - lists[i] = sorted; - rewrite[i] = list_drafts[i] != NULL; - } - /* A cursor or index keeps its position only while its text or list is the - * one it belonged to. */ - for (uint32_t i = 0; i < module->states; ++i) { - if (module->state_type[i] != PURE_STATE_CURSOR && module->state_type[i] != PURE_STATE_INDEX) continue; - uint32_t owner = module->state_ref[i]; - bool same = kept[i] >= 0 && kept[owner] >= 0 && old->state_ref[kept[i]] == (unsigned)kept[owner]; - values[i] = same ? app->values[kept[i]] : 0; - } - if (result == PURE_OK && stored) result = pure_save_open_values(app, module, values, kept, texts, lists); - /* Outputs keep their sends and numbers by name. */ - if (result == PURE_OK) result = old ? remap_outputs(app, old, module, false) : stored ? pure_save_open_links(app, module) : PURE_OK; - for (uint32_t i = 0; i < module->states; ++i) { - uint32_t list = module->state_ref[i]; - if (module->state_type[i] == PURE_STATE_INDEX && moved[list] && values[i] < (double)pure_list_count(lists[list])) - values[i] = (double)moved[list][(size_t)values[i]]; - } - for (uint32_t i = 0; i < module->states; ++i) free(moved[i]); - /* Store the lists whose record or order changed last, so that a failure - * can put every draft back as it was. */ - for (uint32_t i = 0; i < module->states; ++i) - if (rewrite[i]) { - before[i] = pure_draft_version(list_drafts[i]); - pure_draft_hold(list_drafts[i], true); - } - for (uint32_t i = 0; i < module->states && result == PURE_OK; ++i) - if (rewrite[i]) result = pure_save_rewrite_list(module, i, lists[i], list_drafts[i]); - for (uint32_t i = 0; i < module->states; ++i) - if (rewrite[i]) { - if (result != PURE_OK) pure_draft_restore(list_drafts[i], before[i]); - pure_draft_hold(list_drafts[i], false); - } - if (result != PURE_OK) { - for (uint32_t i = 0; i < module->states; ++i) { - if (kept[i] >= 0) { - if (lists[i] && lists[i] != app->lists[kept[i]]) pure_list_free(lists[i]); - continue; - } - pure_draft_close(texts[i]); - pure_list_free(lists[i]); - pure_draft_close(list_drafts[i]); - } - return result; - } - for (uint32_t i = 0; old && i < old->states; ++i) { - bool reused = false, draft_reused = false; - for (uint32_t k = 0; k < module->states; ++k) { - reused = reused || (texts[k] && texts[k] == app->texts[i]) || (lists[k] && lists[k] == app->lists[i]); - draft_reused = draft_reused || (list_drafts[k] && list_drafts[k] == app->list_drafts[i]); - } - if (!reused) { - pure_draft_close(app->texts[i]); - pure_list_free(app->lists[i]); - } - if (!draft_reused) pure_draft_close(app->list_drafts[i]); - } - memcpy(app->values, values, sizeof(values)); - memcpy(app->texts, texts, sizeof(texts)); - memcpy(app->lists, lists, sizeof(lists)); - memcpy(app->list_drafts, list_drafts, sizeof(list_drafts)); - memcpy(app->saved, saved, sizeof(saved)); - if (old) remap_outputs(app, old, module, true); - app->module = module; - ++app->changes; - return PURE_OK; -} - -PURE_COLD PureResult pure_app_start(PureApp *app, const PureModule *module, PureData data) { - *app = (PureApp){.data = data, .fault_pc = UINT32_MAX}; - PureResult result = open_module(app, module); - if (result == PURE_OK) result = pure_app_save(app); - if (result != PURE_OK) pure_app_stop(app); - return result; -} - -PURE_COLD PureResult pure_app_replace(PureApp *app, const PureModule *module) { - pure_app_finish(app); - if (app->open || app->pending_count) { - PureResult saved = pure_app_save(app); /* the burst belongs to the module that made it */ - if (saved != PURE_OK) return saved; - } - return open_module(app, module); -} diff --git a/prototypes/connections/module.h b/prototypes/connections/module.h deleted file mode 100644 index f62326b..0000000 --- a/prototypes/connections/module.h +++ /dev/null @@ -1,343 +0,0 @@ -#ifndef PURE_MODULE_H -#define PURE_MODULE_H -#include "flow.h" -#include "result.h" -#include "module-list.h" -#include "state.h" -#include -#include -#include - -/* Pure module format v5 (docs/design/008-module-format.md). A toolchain such - * as aup produces it; Pure verifies it completely before anything executes: - * operand types (including which text a cursor and which list an index - * belongs to), stack heights and the stack at every join, forward-only - * branches, names, records, tables, constants and effects. */ -enum { - PURE_MODULE_STATES = 32, PURE_MODULE_INPUTS = 32, PURE_MODULE_VIEWS = 64, - PURE_MODULE_HANDLERS = 64, PURE_MODULE_CODE = 65535, PURE_MODULE_STACK = 64, - PURE_MODULE_NAME = 32, PURE_MODULE_STRINGS = 32768, PURE_MODULE_HEADER = 40, - PURE_MODULE_FIELDS = 64, PURE_MODULE_COLLECTIONS = 1, PURE_MODULE_CONSTANTS = 4096, - PURE_MODULE_CONSTANT_BYTES = 1 << 20, PURE_MODULE_RECORDS = 8, PURE_MODULE_RECORD_FIELDS = 64, - PURE_RECORD_FIELDS = 16, PURE_MODULE_JOINS = 8, PURE_APP_PENDING = 256 -}; - -typedef enum { - PURE_OP_CONST, PURE_OP_LOAD, PURE_OP_DT, PURE_OP_ADD, PURE_OP_SUB, PURE_OP_MUL, - PURE_OP_DIV, PURE_OP_NEG, PURE_OP_LT, PURE_OP_LE, PURE_OP_GT, PURE_OP_GE, - PURE_OP_EQ, PURE_OP_NE, PURE_OP_AND, PURE_OP_OR, PURE_OP_NOT, PURE_OP_MIN, - PURE_OP_MAX, PURE_OP_CLAMP, PURE_OP_STORE, PURE_OP_JZ, PURE_OP_JMP, - PURE_OP_FIELD, PURE_OP_DOC, PURE_OP_DOC_SIZE, PURE_OP_LEN, PURE_OP_REPLACE, - PURE_OP_REPLY, PURE_OP_REQUIRE, - PURE_OP_TEXT_CONST, PURE_OP_TEXT_LOAD, PURE_OP_TEXT_INPUT, PURE_OP_NUM_TEXT, - PURE_OP_CONCAT, PURE_OP_CHARS, PURE_OP_LINES, PURE_OP_TEXT_BYTES, - PURE_OP_CUR_LOAD, PURE_OP_CUR_START, PURE_OP_CUR_END, PURE_OP_CUR_MOVE, - PURE_OP_CUR_STORE, PURE_OP_TEXT_INSERT, PURE_OP_TEXT_REMOVE, PURE_OP_TEXT_LINE, - PURE_OP_TEXT_SET, PURE_OP_TEXT_FROM, PURE_OP_BYTES_FROM, PURE_OP_CUR_COUNT, - PURE_OP_LIST_COUNT, PURE_OP_LIST_APPEND, PURE_OP_LIST_INSERT, PURE_OP_LIST_REMOVE, - PURE_OP_ITEM_GET, PURE_OP_ITEM_SET, PURE_OP_INDEX_LOAD, PURE_OP_INDEX_STORE, - PURE_OP_INDEX_MOVE, PURE_OP_INDEX_FIRST, PURE_OP_INDEX_LAST, PURE_OP_INDEX_NUM, PURE_OP_ROW, - PURE_OP_PICK, PURE_OP_SLIDE, PURE_OP_FOLD, PURE_OP_TEXT_COMPARE, PURE_OP_LIST_FIND, PURE_OP_LIST_HAS, - PURE_OP_NOW, PURE_OP_FLOOR, PURE_OP_MOD, PURE_OP_SEND, - PURE_OP_COUNT -} PureOpcode; -/* FOLD's argument: the kind (bits 0-7), the list state (8-15) and the length - * of the block that follows (16-31), run once per item as a column of that - * list. count counts the items whose block leaves a non-zero number; sum, - * max and min take a value and an include flag from their block; first gives - * the index of the first item whose block leaves a non-zero number. */ -typedef enum { PURE_FOLD_COUNT, PURE_FOLD_SUM, PURE_FOLD_MAX, PURE_FOLD_MIN, PURE_FOLD_FIRST } PureFold; -/* TEXT_COMPARE's argument: the relation, in the order of the number - * comparisons (LT LE GT GE EQ NE); texts order by their bytes. */ -typedef enum { PURE_LESS, PURE_LESS_EQUAL, PURE_GREATER, PURE_GREATER_EQUAL, PURE_EQUAL, PURE_NOT_EQUAL } PureRelation; - -/* A timer ticks every `period` milliseconds while a host runs the app. An - * output is a message the app sends (SEND, in a handler; it has fields as a - * message has and no handler): what receives it is the host's wiring. */ -typedef enum { PURE_KIND_KEY = 1, PURE_KIND_MESSAGE = 2, PURE_KIND_TEXT = 3, PURE_KIND_TIMER = 4, - PURE_KIND_OUTPUT = 5 } PureInputKind; -enum { PURE_TIMER_LONGEST = 86400000, PURE_SOURCE = 64 }; -/* A message field: a u32le or a binary64 (fixed, at its offset in the - * payload's prefix), or bytes or text of any length. The prefix also holds - * each variable field's u32le length but the last one's (its offset is - * PURE_FIELD_REST), which takes what follows the others; the variable - * fields' bytes follow the prefix in declaration order. A text is UTF-8 and - * a number finite, or the message fails. */ -typedef enum { PURE_FIELD_U32 = 1, PURE_FIELD_BYTES = 2, PURE_FIELD_NUMBER = 3, PURE_FIELD_TEXT = 4 } PureFieldType; -enum { PURE_FIELD_REST = UINT32_MAX }; -typedef enum { - PURE_STATE_NUM = 0, PURE_STATE_TEXT = 1, PURE_STATE_CURSOR = 2, PURE_STATE_LIST = 3, PURE_STATE_INDEX = 4 -} PureStateType; -/* A cursor that stays before text inserted at it (a mark); other cursors go - * after it (a caret). */ -enum { PURE_CURSOR_STAYS = 1 }; -/* A table view has one row per item of a list, or per item its filter - * keeps; its columns are number or text views evaluated for one row at a - * time. */ -typedef enum { PURE_VIEW_NUM = 0, PURE_VIEW_TEXT = 1, PURE_VIEW_TABLE = 2 } PureViewType; -typedef enum { PURE_UNIT_CHAR = 0, PURE_UNIT_LINE = 1 } PureUnit; -typedef enum { PURE_EDGE_DOWN, PURE_EDGE_UP } PureEdge; -/* EXISTS: an item with that key is already in the keyed list (the event - * reports PURE_CONFLICT). */ -typedef enum { - PURE_FAULT_NONE, PURE_FAULT_DIVIDE, PURE_FAULT_NON_FINITE, PURE_FAULT_CLAMP, - PURE_FAULT_LIMIT, PURE_FAULT_NOT_FOUND, PURE_FAULT_ENCODING, PURE_FAULT_EXISTS -} PureFault; - -typedef struct { double value; uint32_t arg; uint8_t op; } PureOp; -typedef struct { uint32_t start, length; } PureBlock; -typedef struct { uint32_t event; PureBlock code; } PureHandler; -typedef struct { char name[PURE_MODULE_NAME]; uint8_t type; uint32_t offset; } PureField; -typedef struct { - uint8_t kind; - bool keyed; - uint16_t field_start, field_count; - uint32_t prefix; /* fixed payload bytes before the variable fields */ - uint32_t period; /* a timer's, in milliseconds */ -} PureInputShape; - -/* A record type: fields numbered from first, each a number or text. */ -typedef struct { uint16_t first, count; } PureRecord; - -typedef struct { - uint16_t states, inputs, views, handlers, fields, collections, constants, records, record_fields; - uint32_t code_count; - char state_names[PURE_MODULE_STATES][PURE_MODULE_NAME]; - char input_names[PURE_MODULE_INPUTS][PURE_MODULE_NAME]; - char view_names[PURE_MODULE_VIEWS][PURE_MODULE_NAME]; - char collection_names[PURE_MODULE_COLLECTIONS][PURE_MODULE_NAME]; - char input_keys[PURE_MODULE_INPUTS][PURE_MODULE_NAME]; - uint8_t state_type[PURE_MODULE_STATES]; - uint16_t state_ref[PURE_MODULE_STATES]; /* text: constant; cursor: text; list: record; index: list */ - uint8_t state_flags[PURE_MODULE_STATES]; /* cursor: PURE_CURSOR_STAYS */ - /* A keyed list: its key field + 1 (0: none). Its items stay in key order - * (numbers by value, texts by bytes), each key once, and keys do not change. */ - uint8_t state_key[PURE_MODULE_STATES]; - PureInputShape input[PURE_MODULE_INPUTS]; - PureField field[PURE_MODULE_FIELDS]; - double initial[PURE_MODULE_STATES]; - uint8_t view_type[PURE_MODULE_VIEWS]; - uint16_t view_table[PURE_MODULE_VIEWS]; /* a column's table view + 1, or 0 */ - uint16_t view_list[PURE_MODULE_VIEWS]; /* the list of a table or column */ - PureBlock view_code[PURE_MODULE_VIEWS]; - PureHandler handler[PURE_MODULE_HANDLERS]; - /* The code and the text constants, sized to the module in one block - * (storage) that load takes and pure_module_release gives back. */ - PureOp *code; - uint32_t *constant_offset, *constant_length; - unsigned char *constant_data; - void *storage; - char record_names[PURE_MODULE_RECORDS][PURE_MODULE_NAME]; - PureRecord record[PURE_MODULE_RECORDS]; - char record_field_names[PURE_MODULE_RECORD_FIELDS][PURE_MODULE_NAME]; - uint8_t record_field_text[PURE_MODULE_RECORD_FIELDS]; - double record_field_default[PURE_MODULE_RECORD_FIELDS]; /* number, or text constant index */ -} PureModule; - -/* Decode and verify into out, which must be zeroed (a static one is) or hold - * a loaded module: what it held is given back first. On failure out holds - * nothing and error names the rule. */ -PureResult pure_module_load(const void *bytes, size_t size, PureModule *out, - char *error, size_t error_size); -/* Give back a loaded module's code and constants (no app may still run it). */ -void pure_module_release(PureModule *); -/* Read a module file whole and load it (NOT_FOUND when it cannot be read). */ -PureResult pure_module_read(const char *path, PureModule *out, char *error, size_t error_size); -/* Index of a name, or -1. Views are those outside tables. */ -int pure_module_state(const PureModule *, const char *name); -int pure_module_input(const PureModule *, const char *name); -int pure_module_view(const PureModule *, const char *name); -int pure_module_column(const PureModule *, uint32_t table, const char *name); -/* The index state a number view shows as its position (a view that is just - * `at.position`), or -1: what a host moves with pure_app_point. */ -int pure_module_position(const PureModule *, uint32_t view); -/* The text state a text view shows as it is (a view that is just the - * state), or -1: a host may lend its stored extents instead of copying. */ -int pure_module_text_state(const PureModule *, uint32_t view); - -/* Keyed collections live in a data host (for example the tree store). One - * event opens at most one document, the message's key, at its current - * version; edits stay in that working version until the event commits it. - * A version's bytes field is the value's size. */ -typedef struct { - void *context; - PureResult (*open)(void *context, uint32_t collection, const char *key, - void **document, int64_t *revision); - PureVersion (*version)(void *context, void *document); - PureResult (*read)(void *context, void *document, PureVersion, size_t offset, size_t length, - bool (*visit)(void *, PureSpan), void *visit_context); - PureResult (*splice)(void *context, void *document, size_t offset, size_t remove, PureSpan insert); - PureResult (*commit)(void *context, void *document, int64_t *revision); - void (*close)(void *context, void *document); - /* Optional: the host keeps the app's states. state opens a runtime-owned - * value by name as a draft (a missing one is empty at revision 0); save - * publishes the given documents and state drafts together, all or - * nothing, and names each document's new revision. */ - PureResult (*state)(void *context, const char *name, PureDraft **out); - PureResult (*save)(void *context, void *const *documents, size_t document_count, PureDraft *const *states, - size_t count, int64_t *revisions); -} PureData; - -/* A byte or text value inside one event or view: a range of the payload, a - * document or text version, a module constant, a copy or the event's arena. */ -typedef struct { uint8_t source; uint16_t owner; PureVersion version; uint64_t offset, length; } PureBytes; -typedef struct { unsigned char *bytes; size_t used, capacity; } PureArena; - -/* One app instance: its states and at most one open event. Text states are - * runtime-owned drafts; cursors are positions the runtime keeps on character - * boundaries; lists are runtime-owned and an index always names an item - * (or 0 in an empty list). Every event is atomic: on failure nothing changes. - * - * Saving. When the data host keeps states the app starts from the stored - * ones. A save publishes, in one commit, the documents edited by messages and - * the states changed since the last save; if it fails, every event since the - * last save is undone. By default each key, text and message event saves - * before it returns (a frame waits for the next one). Deferred, events apply - * at once and a host saves a burst with one commit: an event is durable once - * a later save succeeds, and each key's document joins a burst once (a second - * message for it, or one past PURE_APP_PENDING keys, saves the burst first). */ -typedef struct { - const PureModule *module; - PureData data; - double values[PURE_MODULE_STATES]; /* numbers, cursor byte positions and list indices */ - PureDraft *texts[PURE_MODULE_STATES]; - PureList *lists[PURE_MODULE_STATES]; - PureUndo undo; /* list changes of the current event */ - PureDraft *list_drafts[PURE_MODULE_STATES]; /* stored lists, when kept */ - PureDraft *stored_values; /* numbers, cursors and indices, when kept */ - PureVersion saved[PURE_MODULE_STATES]; /* text and list versions at the last save */ - /* Events since the last save: what a failed save returns to, and the - * documents messages opened (held until the save). */ - bool deferred, open; - double durable_values[PURE_MODULE_STATES]; - PureVersion durable_texts[PURE_MODULE_STATES], durable_lists[PURE_MODULE_STATES]; - uint64_t durable_ticks, durable_events; - struct PurePending { void *document; char key[65]; bool edited; } *pending; /* grows to PURE_APP_PENDING */ - unsigned pending_count, pending_capacity; - PureFault fault; - uint32_t fault_pc; - uint64_t ticks, events; - /* The time events see as now, in milliseconds since 1970 (UTC). The host - * sets it, so a script can replay any time; views never see it. */ - double now; - uint64_t changes; /* counts every change of states, never back: what a filter's rows are fresh for */ - struct PureRows { uint32_t table; uint64_t changes; size_t *items, count, capacity; } *rows; /* a filtered table's items */ - unsigned row_tables; - /* Current message or text event, kept until pure_app_finish. */ - void *document; - PureSpan payload; - bool has_reply; - PureBytes reply; - unsigned char *copied; - int64_t revision; - PureArena arena, view_arena; - /* Links (runtime-owned, saved with the states): messages sent on outputs - * and not trimmed, in the order sent (the first sent_durable are saved; - * a trimmed one goes at the next save), each output's last number (and - * as saved), those of outputs a module dropped, and the last number - * delivered from each source (and as saved). */ - struct PureSent { uint32_t output; uint64_t seq; unsigned char *payload; size_t size; bool trimmed; } *sent; - size_t sent_count, sent_capacity, sent_durable; - uint64_t last_sent[PURE_MODULE_INPUTS], saved_sent[PURE_MODULE_INPUTS]; - struct PureCarried { char name[PURE_MODULE_NAME]; uint64_t last; } *carried; - unsigned carried_count; - struct PureHeard { char source[PURE_SOURCE + 1]; uint64_t seq, saved; } *heard; - unsigned heard_count, heard_capacity; - PureDraft *stored_links; - bool links_changed; /* since the last save */ - bool duplicate; /* the last delivery was heard before and not applied */ - struct { const char *source; uint64_t seq; bool pending; } delivering; - struct PureSnapshot *snapshot; /* the newest snapshot made, or NULL */ - PureRetired retired; /* list items let go since it was made */ - bool reading; /* a reader: its states are a snapshot's */ -} PureApp; - -PureResult pure_app_start(PureApp *, const PureModule *, PureData data); -/* Publish every change since the last save in one commit, or undo them all. */ -PureResult pure_app_save(PureApp *); -/* Defer saving to pure_app_save (false: save after each event). Needs a data - * host that saves documents together (PureData.save). */ -bool pure_app_defer(PureApp *, bool deferred); -/* Leaves unsaved frame changes unsaved: call pure_app_save first to keep them. */ -void pure_app_stop(PureApp *); -PureResult pure_app_frame(PureApp *); -/* A timer input's tick: an event saved like a key's. */ -PureResult pure_app_tick(PureApp *, uint32_t input); -PureResult pure_app_input(PureApp *, uint32_t input, PureEdge); -/* Point an index state at the item at position (0 first), as a user pointing - * at a row: an event with no handler, saved like a key event. NOT_FOUND past - * the last item, INVALID for a state that is not an index. */ -PureResult pure_app_point(PureApp *, uint32_t state, size_t position); -/* Deliver committed text (for example from an input method). Not UTF-8: invalid. */ -PureResult pure_app_text(PureApp *, uint32_t input, PureSpan text); -/* Deliver a message. base is the revision the sender edited, or -1 for none; - * a handler that edits the document needs one. payload stays alive until - * pure_app_finish. On return app->revision is the document's revision - * (deferred: the revision the next save gives it). */ -PureResult pure_app_message(PureApp *, uint32_t input, const char *key, int64_t base, PureSpan payload); -/* The same, with the variable fields where the host found them: field f - * starts at placed[2f] in payload and has placed[2f + 1] bytes (entries of - * fixed fields are not read, nor are the variable fields' lengths in the - * prefix). A host that parsed a request in place passes the fields' bytes - * where they are. */ -PureResult pure_app_message_fields(PureApp *, uint32_t input, const char *key, int64_t base, PureSpan payload, - const uint32_t *placed); -/* Visit the reply of the last message in order, without copying, until the - * next event (deferred: copy it before a later save too). */ -PureResult pure_app_reply(const PureApp *, bool (*visit)(void *, PureSpan), void *context); -void pure_app_finish(PureApp *); -/* A number view; false for a text view or a failed evaluation. */ -bool pure_app_view(PureApp *, uint32_t view, double *out); -/* A text view, lent in order until the next view evaluation. */ -PureResult pure_app_view_text(PureApp *, uint32_t view, bool (*visit)(void *, PureSpan), void *context); -/* The rows of a table view (its list's items, or those its filter keeps), - * or 0 for another view. A host evaluates only the cells it shows. */ -size_t pure_app_rows(PureApp *, uint32_t table); -/* The item a row of a table shows, and the row that shows an item (false: - * its filter leaves the item out). Without a filter both are the identity. */ -bool pure_app_row_item(PureApp *, uint32_t table, size_t row, size_t *item); -bool pure_app_item_row(PureApp *, uint32_t table, size_t item, size_t *row); -bool pure_app_cell(PureApp *, uint32_t column, size_t row, double *out); -PureResult pure_app_cell_text(PureApp *, uint32_t column, size_t row, bool (*visit)(void *, PureSpan), void *context); -/* Switch to another verified module. States keep their value by name (text - * states keep their content, cursors their position in that text); new states - * start at their initial value. The module must outlive the app. */ -PureResult pure_app_replace(PureApp *, const PureModule *); - -/* Connections between apps (docs/research/078-app-connections.md). A host - * sees only saved sends, delivers each at least once, in order per output, - * and trims what its receiver has saved; a receiver applies each once. */ -typedef bool (*PureSentVisit)(void *context, uint32_t output, uint64_t seq, PureSpan payload); -void pure_app_outbox(const PureApp *, PureSentVisit visit, void *context); -/* The same, newest first (a host looking only for what is new stops early). */ -void pure_app_outbox_newest(const PureApp *, PureSentVisit visit, void *context); -/* Forget output's saved sends numbered up to through (saved with the next save). */ -void pure_app_trim(PureApp *, uint32_t output, uint64_t through); -/* Deliver a message event from source (a sender's name, 1 to 64 printable - * ASCII bytes) numbered seq (from 1, growing per source). A number heard - * before is not applied again: it succeeds with app->duplicate set. The - * number counts as heard with the event's changes, and also when the app - * refuses the message (a fault: it was delivered, and refused); not when a - * save fails. placed is as for pure_app_message_fields, or NULL. */ -PureResult pure_app_deliver(PureApp *, uint32_t input, const char *source, uint64_t seq, PureSpan payload, - const uint32_t *placed); - -/* Readers on other threads. Right after a save an app can make a snapshot of - * its saved states, for evaluating views anywhere while it keeps applying - * events: numbers are copied; text states share their stored versions and - * lists their items (a list unchanged since the last snapshot shares that - * snapshot's copy), so the cost does not grow with the text. NULL when a - * change waits for a save, when a text state is not all stored (for example - * without a data host that keeps states), or when memory runs out. The caller - * gets a reference; the app keeps its own until the next snapshot. */ -typedef struct PureSnapshot PureSnapshot; -PureSnapshot *pure_app_snapshot(PureApp *); -void pure_snapshot_hold(PureSnapshot *); -/* On any thread; the last reference frees it. */ -void pure_snapshot_release(PureSnapshot *); -/* Show a snapshot's states in reader, a zeroed app or a reader before: its - * views read them while the snapshot is held, with the reader's own arena and - * row cache (one thread at a time). It applies no events. pure_app_stop frees - * the reader and leaves the snapshot alone. */ -void pure_app_read(PureApp *reader, const PureSnapshot *); -#endif diff --git a/prototypes/connections/package_source.py b/prototypes/connections/package_source.py index 4a2487c..fa0fb4a 100644 --- a/prototypes/connections/package_source.py +++ b/prototypes/connections/package_source.py @@ -10,7 +10,7 @@ import tarfile ROOT = Path(__file__).resolve().parents[2] PREFIX = 'pure-source-preview' -SOURCE_ROOTS = ('prototypes/connections', 'prototypes/workbench', 'prototypes/common') +SOURCE_ROOTS = ('core', 'store', 'serve', 'cli', 'prototypes/connections', 'prototypes/workbench', 'prototypes/common') SKIP_DIRS = {'build', 'results', '__pycache__', 'node_modules', '.git', '.playwright-cli'} SUFFIXES = {'.c', '.h', '.py', '.mjs', '.js', '.zig', '.json', '.pure', '.html', '.css', '.svg', '.wasm', '.md', '.txt'} diff --git a/prototypes/connections/pure-run.c b/prototypes/connections/pure-run.c deleted file mode 100644 index cfc3b5c..0000000 --- a/prototypes/connections/pure-run.c +++ /dev/null @@ -1,196 +0,0 @@ -#define _POSIX_C_SOURCE 200809L -#include "module-store.h" -#include "store.h" -#include -#include -#include - -/* Test host for a verified module: reads commands from stdin and prints one - * JSON snapshot per command. Not a product interface. With --store the app's - * states live in that tree store: it starts from them, and frame changes are - * saved at the end of input or by save. --batch N defers saving and saves - * every N commands (and at the end), so N events share one commit. - * down NAME | up NAME | frame [N] | type NAME HEX|- | tick NAME | time MS | replace PATH | save | show - * Text states and text views print as JSON strings; cursors as byte positions; - * lists as arrays of objects; indices as positions; a table view as an array - * with an object of its columns per row. A failed view or cell is null. */ -static const char *const fault_names[] = {"none", "divide", "non_finite", "clamp", "limit", "not_found", "encoding", "exists"}; -/* time MS sets the time events see (milliseconds since 1970); tick NAME is a timer's tick. */ - -static bool load(const char *path, PureModule *module) { - char error[160]; - if (pure_module_read(path, module, error, sizeof(error)) == PURE_OK) return true; - fprintf(stderr, "%s: %s\n", path, error); - return false; -} - -static bool json_span(void *context, PureSpan span) { - (void)context; - for (size_t i = 0; i < span.size; ++i) { - unsigned char c = span.data[i]; - if (c == '"' || c == '\\') printf("\\%c", c); - else if (c < 0x20) printf("\\u%04x", c); - else putchar(c); - } - return true; -} - -/* A text view is collected first: it may fail after some bytes. */ -typedef struct { unsigned char bytes[1 << 16]; size_t used; bool fits; } Collected; - -static bool collect(void *context, PureSpan span) { - Collected *c = context; - c->fits = c->fits && span.size <= sizeof(c->bytes) - c->used; - if (c->fits && span.size) memcpy(c->bytes + c->used, span.data, span.size); - if (c->fits) c->used += span.size; - return true; -} - -static void show_list(const PureModule *m, const PureList *list, const PureRecord *record) { - putchar('['); - for (size_t at = 0; at < pure_list_count(list); ++at) { - printf("%s{", at ? "," : ""); - for (unsigned f = 0; f < record->count; ++f) { - PureCell cell = pure_list_get(list, at, f); - printf("%s\"%s\":", f ? "," : "", m->record_field_names[record->first + f]); - if (!m->record_field_text[record->first + f]) printf("%.17g", cell.num); - else { - putchar('"'); - json_span(NULL, (PureSpan){cell.text, cell.size}); - putchar('"'); - } - } - putchar('}'); - } - putchar(']'); -} - -/* A number or text view, or a column at a row. */ -static void show_value(PureApp *app, uint32_t view, size_t row) { - const PureModule *m = app->module; - static Collected text; - text.used = 0; - text.fits = true; - double value; - bool column = m->view_table[view]; - if (m->view_type[view] == PURE_VIEW_TEXT) { - PureResult result = column ? pure_app_cell_text(app, view, row, collect, &text) - : pure_app_view_text(app, view, collect, &text); - if (result != PURE_OK || !text.fits) printf("null"); - else { - putchar('"'); - json_span(NULL, (PureSpan){text.bytes, text.used}); - putchar('"'); - } - } else if (column ? pure_app_cell(app, view, row, &value) : pure_app_view(app, view, &value)) printf("%.17g", value); - else printf("null"); -} - -static void show(PureApp *app, const char *failure) { - const PureModule *m = app->module; - printf("{\"states\":{"); - for (uint32_t i = 0; i < m->states; ++i) { - printf("%s\"%s\":", i ? "," : "", m->state_names[i]); - if (m->state_type[i] == PURE_STATE_TEXT) { - putchar('"'); - pure_draft_read(app->texts[i], 0, pure_draft_size(app->texts[i]), json_span, NULL); - putchar('"'); - } else if (m->state_type[i] == PURE_STATE_LIST) show_list(m, app->lists[i], &m->record[m->state_ref[i]]); - else printf("%.17g", app->values[i]); - } - printf("},\"views\":{"); - bool first = true; - for (uint32_t i = 0; i < m->views; ++i) { - if (m->view_table[i]) continue; - printf("%s\"%s\":", first ? "" : ",", m->view_names[i]); - first = false; - if (m->view_type[i] != PURE_VIEW_TABLE) { show_value(app, i, 0); continue; } - putchar('['); - for (size_t row = 0; row < pure_app_rows(app, i); ++row) { - printf("%s{", row ? "," : ""); - for (uint32_t c = 0, shown = 0; c < m->views; ++c) { - if (m->view_table[c] != i + 1) continue; - printf("%s\"%s\":", shown++ ? "," : "", m->view_names[c]); - show_value(app, c, row); - } - putchar('}'); - } - putchar(']'); - } - printf("},\"tick\":%llu,\"successfulEvents\":%llu,\"fault\":\"%s\",\"error\":%s%s%s}\n", - (unsigned long long)app->ticks, (unsigned long long)app->events, fault_names[app->fault], - failure ? "\"" : "", failure ? failure : "null", failure ? "\"" : ""); -} - -static int hex(char c) { - return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1; -} - -int main(int argc, char **argv) { - static PureModule modules[2]; - bool stored = argc >= 4 && !strcmp(argv[2], "--store"); - long batch = argc == 6 && stored && !strcmp(argv[4], "--batch") ? strtol(argv[5], NULL, 10) : 0; - bool usage = argc != 2 && !(stored && argc == 4) && !(batch > 0 && batch <= 100000); - if (usage || !load(argv[1], &modules[0])) { - if (usage) fputs("pure-run MODULE [--store PATH [--batch N]] < commands\n", stderr); - return 2; - } - if (stored && pure_store_init(argv[3]) != PURE_OK) { fprintf(stderr, "cannot open store %s\n", argv[3]); return 2; } - unsigned active = 0; - PureApp app; - PureResult started = pure_app_start(&app, &modules[active], stored ? pure_tree_data(argv[3]) : (PureData){0}); - if (started != PURE_OK) { fprintf(stderr, "cannot start (%d)\n", (int)started); return 2; } - if (batch && !pure_app_defer(&app, true)) { fputs("cannot defer saving\n", stderr); return 2; } - long unsaved = 0; - static char line[1 << 16]; - static unsigned char text[1 << 15]; - int status = 0; - while (fgets(line, sizeof(line), stdin)) { - char command[16] = "", argument[256] = "", data[sizeof(line)] = ""; - const char *failure = NULL; - line[strcspn(line, "\r\n")] = 0; - if (sscanf(line, "%15s %255s %65535s", command, argument, data) < 1) continue; - if (!strcmp(command, "down") || !strcmp(command, "up")) { - int input = pure_module_input(app.module, argument); - if (input < 0) failure = "unknown input"; - else if (pure_app_input(&app, (uint32_t)input, command[0] == 'd' ? PURE_EDGE_DOWN : PURE_EDGE_UP) != PURE_OK) - failure = "event failed"; - } else if (!strcmp(command, "type")) { - int input = pure_module_input(app.module, argument); - size_t length = strcmp(data, "-") ? strlen(data) / 2 : 0; - for (size_t i = 0; i < length && length * 2 == strlen(data); ++i) - text[i] = (unsigned char)(hex(data[2 * i]) << 4 | hex(data[2 * i + 1])); - if (input < 0) failure = "unknown input"; - else if (pure_app_text(&app, (uint32_t)input, (PureSpan){text, length}) != PURE_OK) failure = "event failed"; - } else if (!strcmp(command, "tick")) { - int input = pure_module_input(app.module, argument); - if (input < 0) failure = "unknown input"; - else if (pure_app_tick(&app, (uint32_t)input) != PURE_OK) failure = "event failed"; - } else if (!strcmp(command, "time")) { - char *end; - double now = strtod(argument, &end); - if (!*argument || *end || !(now >= 0 && now < 1e15)) failure = "time"; - else app.now = now; - } else if (!strcmp(command, "frame")) { - long count = *argument ? strtol(argument, NULL, 10) : 1; - if (count < 1 || count > 600) failure = "frame count"; - for (long i = 0; !failure && i < count; ++i) - if (pure_app_frame(&app) != PURE_OK) failure = "event failed"; - } else if (!strcmp(command, "replace")) { - if (!load(argument, &modules[!active])) failure = "invalid module"; - else if (pure_app_replace(&app, &modules[!active]) != PURE_OK) failure = "replace failed"; - else active = !active; - } else if (!strcmp(command, "save")) { - if (pure_app_save(&app) != PURE_OK) failure = "save failed"; - } else if (strcmp(command, "show")) failure = "unknown command"; - if (batch && strcmp(command, "show") && ++unsaved == batch) { - unsaved = 0; - if (pure_app_save(&app) != PURE_OK) failure = "save failed"; - } - if (failure) status = 1; - show(&app, failure); - } - if (pure_app_save(&app) != PURE_OK) { fputs("cannot save\n", stderr); status = 1; } - pure_app_stop(&app); - return status; -} diff --git a/prototypes/connections/result.h b/prototypes/connections/result.h deleted file mode 100644 index 8abb0f2..0000000 --- a/prototypes/connections/result.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef PURE_RESULT_H -#define PURE_RESULT_H -typedef enum { - PURE_OK, PURE_NOT_FOUND, PURE_CONFLICT, PURE_BUSY, PURE_BUDGET, - PURE_LIMIT, PURE_STORE_ERROR, PURE_UNCERTAIN, - PURE_INVALID, PURE_DENIED, PURE_CLOSED, PURE_CANCELLED, PURE_ALLOCATION -} PureResult; -#endif diff --git a/prototypes/connections/runtime.c b/prototypes/connections/runtime.c deleted file mode 100644 index 4a857c7..0000000 --- a/prototypes/connections/runtime.c +++ /dev/null @@ -1,394 +0,0 @@ -#include "runtime.h" -#include - -#define NONE SIZE_MAX -enum { FREE, RECEIVING, READY, RUNNING, DONE }; -#ifdef PURE_RUNTIME_METRICS -#define METRIC(...) do { __VA_ARGS__; } while (0) -#else -#define METRIC(...) ((void)0) -#endif - -static PureWork *lookup(PureTicket ticket, bool require_held) { - PureRuntime *r = ticket.runtime; - if (!r || ticket.slot >= r->capacity || !ticket.generation) return NULL; - PureWork *w = &r->work[ticket.slot]; - if (w->state == FREE || w->generation != ticket.generation || - (require_held && !w->held)) return NULL; - return w; -} - -static bool valid_key(const char *key) { - if (!key || !*key) return false; - size_t n = 0; - for (; n <= 64 && key[n]; ++n) { - unsigned char c = (unsigned char)key[n]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || c == '-' || c == '_')) return false; - } - return n <= 64; -} - -/* WRITE results are loans from the admitted input, not a second allocation. - * Integer offsets avoid relational comparisons between unrelated C pointers. */ -static bool borrowed_reply(PureSpan input, PureSpan reply) { - if (!reply.size) return true; - if (!input.data || !reply.data) return false; - uintptr_t base = (uintptr_t)input.data, start = (uintptr_t)reply.data; - return start >= base && start - base <= input.size && - reply.size <= input.size - (start - base); -} - -static void dispose(PureRuntime *r, PureResource *resource) { - PureResource old = *resource; - *resource = (PureResource){0}; - if (old.release) { - bool running = r->running; - r->running = true; - old.release(old.allocation, old.allocation_size); - r->running = running; - } -} - -static void payload_release(PureRuntime *r, PureWork *w); - -/* Iteratively retire a chain of forwarded values. An arbitrarily long chain - * must not turn completion into recursive stack growth. */ -static void retire(PureRuntime *r, PureWork *w) { - while (w && !w->held && !w->pins) { - PureTicket parent = w->parent; -#ifdef PURE_RUNTIME_METRICS - bool released_payload = w->charged || w->resource.release || parent.runtime; -#endif - dispose(r, &w->resource); - r->retained -= w->charged; - METRIC(if (released_payload) ++r->releases); - size_t slot = (size_t)(w - r->work); - *w = (PureWork){.next = r->free_head}; - r->free_head = slot; - --r->active; - if (!parent.runtime) break; - r = parent.runtime; - w = lookup(parent, false); - /* Pins can only be created and removed by the owner-thread core. */ - if (!w || !w->pins) break; - --w->pins; - } -} - -static void payload_release(PureRuntime *r, PureWork *w) { - PureTicket parent = w->parent; -#ifdef PURE_RUNTIME_METRICS - bool released_payload = w->charged || w->resource.release || parent.runtime; -#endif - dispose(r, &w->resource); - r->retained -= w->charged; - METRIC(if (released_payload) ++r->releases); - w->charged = 0; - w->parent = (PureTicket){0}; - if (parent.runtime) { - PureWork *source = lookup(parent, false); - if (source && source->pins) { --source->pins; retire(parent.runtime, source); } - } -} - -bool pure_runtime_init(PureRuntime *r, PureWork *work, size_t capacity, - const void *definition, PureHost host, size_t budget) { - if (!r || ((work == NULL) != (capacity == 0))) return false; - *r = (PureRuntime){.definition = definition, .host = host, .free_head = NONE, - .ready_head = NONE, .ready_tail = NONE, - .budget = budget, .accepting = true}; - return pure_runtime_bind(r, work, capacity); -} - -bool pure_runtime_bind(PureRuntime *r, PureWork *work, size_t capacity) { - if (!r || r->running || r->active || r->retained || - r->ready_head != NONE || r->ready_tail != NONE || - ((work == NULL) != (capacity == 0))) - return false; - r->work = work; - r->capacity = capacity; - r->free_head = capacity ? 0 : NONE; - for (size_t i = 0; i < capacity; ++i) - work[i] = (PureWork){.next = i + 1 < capacity ? i + 1 : NONE}; - return true; -} - -bool pure_runtime_restart(PureRuntime *r, const void *definition, PureHost host, size_t budget) { - if (!r || r->running || r->active) return false; - r->definition = definition; - r->host = host; - r->budget = budget; - r->accepting = true; - return true; -} - -void pure_runtime_stop(PureRuntime *r) { if (r && !r->running) r->accepting = false; } - -static PureResult admission(PureRuntime *r, PureInput input, const char *key, - int64_t expected, size_t bytes) { - if (!r || r->running || !valid_key(key) || - (input != PURE_INPUT_READ && input != PURE_INPUT_WRITE) || - (input == PURE_INPUT_READ ? bytes != 0 : expected < 0)) return PURE_INVALID; - if (!r->accepting) return PURE_CLOSED; - bool missing_effect = input == PURE_INPUT_READ ? !r->host.read : !r->host.write; -#ifdef PURE_ASYNC_EFFECTS - if (missing_effect && !r->async_effects) return PURE_DENIED; -#else - if (missing_effect) return PURE_DENIED; -#endif - if (r->free_head == NONE) return PURE_BUSY; - if (r->serial == UINT64_MAX) return PURE_LIMIT; - if (bytes > r->budget - r->retained) { METRIC(++r->budget_rejections); return PURE_BUDGET; } - return PURE_OK; -} - -static PureTicket occupy(PureRuntime *r, PureInput input, const char *key, - int64_t expected, size_t bytes, PureResource resource) { - size_t slot = r->free_head; - PureWork *w = &r->work[slot]; - r->free_head = w->next; - *w = (PureWork){.generation = ++r->serial, .next = NONE, .charged = bytes, - .state = RECEIVING, .held = true, .input = input, .expected = expected, - .resource = resource}; - memcpy(w->key, key, strlen(key) + 1); - ++r->active; - METRIC(++r->accepted); - r->retained += bytes; - METRIC(if (r->active > r->peak_active) r->peak_active = r->active); - METRIC(if (r->retained > r->peak_retained) r->peak_retained = r->retained); - return (PureTicket){r, slot, w->generation}; -} - -bool pure_runtime_ready(PureTicket ticket) { - PureWork *w = lookup(ticket, true); - if (!w || ticket.runtime->running || w->state != RECEIVING) return false; - PureRuntime *r = ticket.runtime; - w->state = READY; - if (r->ready_tail == NONE) r->ready_head = ticket.slot; - else r->work[r->ready_tail].next = ticket.slot; - r->ready_tail = ticket.slot; - return true; -} - -PureResult pure_runtime_accept(PureRuntime *r, PureInput input, const char *key, - int64_t expected, size_t size, PureTicket *out) { - if (!out) return PURE_INVALID; - PureResult status = admission(r, input, key, expected, size); - if (status != PURE_OK) return status; - PureResource resource = {0}; - if (size) { - if (!r->host.allocate) return PURE_ALLOCATION; - r->running = true; - bool ok = r->host.allocate(r->host.context, size, &resource); - r->running = false; - if (!ok || !resource.body.data || resource.body.size != size || !resource.release) { - dispose(r, &resource); - return PURE_ALLOCATION; - } - } - *out = occupy(r, input, key, expected, size, resource); - if (input == PURE_INPUT_READ) pure_runtime_ready(*out); - return PURE_OK; -} - -unsigned char *pure_runtime_ingress(PureTicket ticket, size_t *size) { - PureWork *w = lookup(ticket, true); - if (size) *size = 0; - if (!w || ticket.runtime->running || w->state != RECEIVING || - w->input != PURE_INPUT_WRITE || w->parent.runtime) return NULL; - if (size) *size = w->resource.body.size; - return (unsigned char *)w->resource.body.data; -} - -bool pure_runtime_step(PureRuntime *r) { - if (!r || r->running || r->ready_head == NONE) return false; -#ifdef PURE_ASYNC_EFFECTS - PureWork *head = &r->work[r->ready_head]; - if (head->input == PURE_INPUT_READ ? !r->host.read : !r->host.write) return false; -#endif - size_t slot = r->ready_head; - PureWork *w = &r->work[slot]; - r->ready_head = w->next; - if (r->ready_head == NONE) r->ready_tail = NONE; - w->next = NONE; - w->state = RUNNING; - r->running = true; - PureOutcome outcome = {0}; - if (w->input == PURE_INPUT_WRITE) { - PureSpan reply = {0}; - outcome.status = r->host.write(r->host.context, r->definition, w->key, - w->expected, w->resource.body, &reply, - &outcome.revision); - if (outcome.status == PURE_OK) { - if (borrowed_reply(w->resource.body, reply)) outcome.body = reply; - else outcome.status = PURE_STORE_ERROR; - } - METRIC(if (outcome.status == PURE_OK) ++r->commits; - else if (outcome.status == PURE_CONFLICT) ++r->conflicts); - } else { - outcome.status = r->host.read(r->host.context, r->definition, w->key, - r->budget - r->retained, &w->resource, &outcome.revision); - if (outcome.status == PURE_OK) { - if (w->resource.body.size > r->budget - r->retained) outcome.status = PURE_BUDGET; - else if (w->resource.body.size && !w->resource.release) - outcome.status = PURE_STORE_ERROR; - else { - w->charged = w->resource.body.size; - r->retained += w->charged; - METRIC(if (r->retained > r->peak_retained) r->peak_retained = r->retained); - outcome.body = w->resource.body; - METRIC(++r->gets); - } - } - METRIC(if (outcome.status == PURE_BUDGET) ++r->budget_rejections); - } - if (outcome.status != PURE_OK || !outcome.body.size) { - outcome.body = (PureSpan){0}; - payload_release(r, w); - } - METRIC(r->borrowed_output_bytes += outcome.body.size); - w->outcome = outcome; - w->state = DONE; - r->running = false; - return true; -} - -#ifdef PURE_ASYNC_EFFECTS -bool pure_runtime_enable_async(PureRuntime *r) { - if (!r || r->running || r->active) return false; - r->async_effects = true; - return true; -} - -PureTake pure_runtime_take(PureRuntime *r, PureEffect *out) { - if (!r || !out || !r->async_effects || r->running || r->ready_head == NONE) - return PURE_TAKE_EMPTY; - size_t slot = r->ready_head; - PureWork *w = &r->work[slot]; - r->ready_head = w->next; - if (r->ready_head == NONE) r->ready_tail = NONE; - w->next = NONE; - w->state = RUNNING; - *out = (PureEffect){.ticket = {r, slot, w->generation}, .input = w->input, - .key = w->key, .expected = w->expected, - .value = w->resource.body, - .available = r->budget - r->retained}; - return PURE_TAKE_EFFECT; -} - -bool pure_runtime_complete(PureTicket ticket, PureResult status, - PureResource *read_resource, PureSpan write_reply, - int64_t revision) { - PureWork *w = lookup(ticket, false); - PureRuntime *r = ticket.runtime; - if (!w || r->running || w->state != RUNNING || - (w->input == PURE_INPUT_WRITE && read_resource) || - (w->input == PURE_INPUT_READ && write_reply.size) || - (w->input == PURE_INPUT_WRITE && status == PURE_OK && - !borrowed_reply(w->resource.body, write_reply)) || - (read_resource && !read_resource->release && - (read_resource->body.size || read_resource->allocation))) return false; - r->running = true; - PureOutcome outcome = {.status = status, .revision = revision}; - if (w->input == PURE_INPUT_READ) { - PureResource acquired = {0}; - if (read_resource) { - acquired = *read_resource; - *read_resource = (PureResource){0}; - } - if (outcome.status == PURE_OK) { - if (acquired.body.size > r->budget - r->retained) - outcome.status = PURE_BUDGET; - else if (acquired.body.size && !acquired.release) - outcome.status = PURE_STORE_ERROR; - else { - w->resource = acquired; - acquired = (PureResource){0}; - w->charged = w->resource.body.size; - r->retained += w->charged; - METRIC(if (r->retained > r->peak_retained) r->peak_retained = r->retained); - outcome.body = w->resource.body; - METRIC(++r->gets); - } - } - dispose(r, &acquired); - METRIC(if (outcome.status == PURE_BUDGET) ++r->budget_rejections); - } else { - if (status == PURE_OK) outcome.body = write_reply; - METRIC(if (outcome.status == PURE_OK) ++r->commits; - else if (outcome.status == PURE_CONFLICT) ++r->conflicts); - } - if (outcome.status != PURE_OK || !outcome.body.size) { - outcome.body = (PureSpan){0}; - payload_release(r, w); - } - METRIC(r->borrowed_output_bytes += outcome.body.size); - w->outcome = outcome; - w->state = DONE; - r->running = false; - if (!w->held) retire(r, w); - return true; -} -#endif - -bool pure_runtime_result(PureTicket ticket, PureOutcome *out) { - PureWork *w = lookup(ticket, true); - if (!out || !w || ticket.runtime->running || w->state != DONE) return false; - *out = w->outcome; - return true; -} - -PureResult pure_runtime_forward(PureTicket source, PureRuntime *r, - const char *key, int64_t expected, PureTicket *out) { - PureWork *parent = lookup(source, true); - if (!out || !parent || source.runtime->running || parent->state != DONE || - parent->outcome.status != PURE_OK || (parent->outcome.body.size && !parent->outcome.body.data)) - return PURE_INVALID; - PureResult status = admission(r, PURE_INPUT_WRITE, key, expected, parent->outcome.body.size); - if (status != PURE_OK) return status; - if (parent->pins == SIZE_MAX) return PURE_LIMIT; - PureTicket ticket = occupy(r, PURE_INPUT_WRITE, key, expected, parent->outcome.body.size, - (PureResource){.body = parent->outcome.body}); - ++parent->pins; - r->work[ticket.slot].parent = source; - pure_runtime_ready(ticket); - *out = ticket; - return PURE_OK; -} - -static void unlink_ready(PureRuntime *r, size_t slot) { - size_t previous = NONE; - for (size_t at = r->ready_head; at != NONE; at = r->work[at].next) { - if (at == slot) { - if (previous == NONE) r->ready_head = r->work[at].next; - else r->work[previous].next = r->work[at].next; - if (r->ready_tail == at) r->ready_tail = previous; - r->work[at].next = NONE; - return; - } - previous = at; - } -} - -bool pure_runtime_cancel(PureTicket ticket) { - PureWork *w = lookup(ticket, true); - if (!w || ticket.runtime->running || (w->state != RECEIVING && w->state != READY)) return false; - if (w->state == READY) unlink_ready(ticket.runtime, ticket.slot); - payload_release(ticket.runtime, w); - w->outcome = (PureOutcome){.status = PURE_CANCELLED}; - w->state = DONE; - return true; -} - -bool pure_runtime_release(PureTicket ticket) { - PureWork *w = lookup(ticket, true); - if (!w || ticket.runtime->running) return false; - if (w->state == READY) unlink_ready(ticket.runtime, ticket.slot); - w->held = false; -#ifdef PURE_ASYNC_EFFECTS - if (w->state == RUNNING) return true; -#endif - retire(ticket.runtime, w); - return true; -} diff --git a/prototypes/connections/runtime.h b/prototypes/connections/runtime.h deleted file mode 100644 index 70f5d08..0000000 --- a/prototypes/connections/runtime.h +++ /dev/null @@ -1,150 +0,0 @@ -#ifndef PURE_RUNTIME_H -#define PURE_RUNTIME_H -#include "flow.h" -#include "result.h" -#include - -/* The optional metrics profile changes PureRuntime's layout. Give its init - * function a distinct link name so mixed-profile objects cannot link quietly. */ -#ifdef PURE_RUNTIME_METRICS -#define pure_runtime_init pure_runtime_init_metrics -#endif - -/* One owner thread controls admission, payload credits, completion and loans. - * An application definition is opaque to this core. Hosts interpret it and - * perform effects; no parser, storage, socket or OS type is linked here. */ -typedef struct PureRuntime PureRuntime; -typedef struct PureWork PureWork; -typedef struct { PureRuntime *runtime; size_t slot; uint64_t generation; } PureTicket; -typedef enum { PURE_INPUT_READ, PURE_INPUT_WRITE } PureInput; - -/* The host owns this resource until it successfully returns it to the core. - * release must end all access and free/unmap allocation exactly once. It runs - * while that domain is busy; calls back into the same domain are rejected. - * A resource the host sends in pieces (such as extents of a mapping) has no - * contiguous body: body.data is NULL and body.size is still what it holds and - * is charged. Its outcome has no body to read and cannot be forwarded. */ -typedef struct { - PureSpan body; - void *allocation; - size_t allocation_size; - void (*release)(void *allocation, size_t size); -} PureResource; -typedef struct { - void *context; - bool (*allocate)(void *, size_t, PureResource *); - PureResult (*read)(void *, const void *definition, const char *, - size_t available, PureResource *, int64_t *revision); - /* A successful reply may borrow only from input. The runtime retains its - * backing until all reply consumers finish. Effect errors publish no reply. */ - PureResult (*write)(void *, const void *definition, const char *, - int64_t expected, PureSpan input, PureSpan *reply, - int64_t *revision); -} PureHost; -typedef struct { PureResult status; int64_t revision; PureSpan body; } PureOutcome; - -/* Caller-provided work slots: no background thread or per-domain heap. These - * records are implementation storage, not an ABI or a wire representation. */ -/* Internal storage order packs the fixed key around small flags. This is not - * a public ABI; the layout is checked as part of the core cost gate. */ -struct PureWork { - PureTicket parent; - unsigned state; - PureInput input; - bool held; - char key[65]; - int64_t expected; - PureOutcome outcome; - PureResource resource; - uint64_t generation; - size_t next, charged, pins; -}; -struct PureRuntime { - const void *definition; - PureHost host; - PureWork *work; - size_t capacity, free_head, ready_head, ready_tail; - size_t budget, retained; -#ifdef PURE_RUNTIME_METRICS - size_t peak_retained; -#endif - size_t active; -#ifdef PURE_RUNTIME_METRICS - size_t peak_active; -#endif - uint64_t serial; -#ifdef PURE_RUNTIME_METRICS - uint64_t accepted, commits, gets, conflicts, budget_rejections; - uint64_t releases, borrowed_output_bytes; -#endif - bool accepting, running, async_effects; -}; - -/* Enforce repeated storage cost on 64-bit C hosts, including macOS where the - * Linux object-section checker is unavailable. */ -#if SIZE_MAX == UINT64_MAX && UINTPTR_MAX == UINT64_MAX -_Static_assert(sizeof(PureWork) == 216, "work-slot layout exceeded its reviewed size"); -#ifdef PURE_RUNTIME_METRICS -_Static_assert(sizeof(PureRuntime) == 192, "metrics domain layout changed"); -#else -_Static_assert(sizeof(PureRuntime) == 120, "domain layout changed"); -#endif -#endif - -/* Initialize fresh storage once. A NULL work pointer with capacity zero starts - * an idle domain; bind work before admission. Use restart, not init, to reuse - * a domain so its monotonic ticket sequence survives restart. definition is - * host-owned and remains alive until all work has drained. */ -bool pure_runtime_init(PureRuntime *, PureWork *, size_t capacity, - const void *definition, PureHost, size_t budget); -bool pure_runtime_restart(PureRuntime *, const void *definition, PureHost, size_t budget); -void pure_runtime_stop(PureRuntime *); /* stop admission, allow queued work to drain */ -/* Replace caller-owned work storage only after all tickets and loans retire. - * A zero-capacity binding releases idle slot storage without resetting budget, - * ticket generation or optional metrics. The caller retains the bound array's life. */ -bool pure_runtime_bind(PureRuntime *, PureWork *, size_t capacity); - -/* READ queues immediately. WRITE reserves its slot, budget and host allocation - * before returning. Fill ingress then publish once with ready. A partial input - * released without ready never executes. On error, out is unchanged. */ -PureResult pure_runtime_accept(PureRuntime *, PureInput, const char *key, - int64_t expected, size_t input_size, PureTicket *out); -unsigned char *pure_runtime_ingress(PureTicket, size_t *size); -bool pure_runtime_ready(PureTicket); -bool pure_runtime_step(PureRuntime *); /* execute at most one ready job */ -bool pure_runtime_result(PureTicket, PureOutcome *out); - -/* Pin a completed reply and queue it as another domain's WRITE input, with no - * allocation, serialization or payload copy. Source may be released as soon - * as this succeeds; the core retains its backing until the consumer ends. - * Each domain charges its own retained input bytes as admission credits. - * Credits across domains are NOT distinct physical memory and must not be summed. */ -PureResult pure_runtime_forward(PureTicket source, PureRuntime *destination, - const char *key, int64_t expected, PureTicket *out); -bool pure_runtime_cancel(PureTicket); /* only before execution; keeps a completion */ -bool pure_runtime_release(PureTicket); /* invalidates caller's ticket, preserves pins */ - -#ifdef PURE_ASYNC_EFFECTS -/* Optional owner-thread effect suspension. Enable before accepting work. The - * host may complete effects in any order, but must call complete on the owner - * thread after workers stop accessing its borrowed spans. An in-flight effect - * keeps its input alive even if the caller releases the ticket. Effects - * borrowed by take stay valid until completion. */ -typedef enum { PURE_TAKE_EMPTY, PURE_TAKE_EFFECT } PureTake; -typedef struct { - PureTicket ticket; - PureInput input; - const char *key; - int64_t expected; - PureSpan value; /* raw ingress for WRITE; empty for READ */ - size_t available; /* remaining payload credits for READ */ -} PureEffect; -bool pure_runtime_enable_async(PureRuntime *); -PureTake pure_runtime_take(PureRuntime *, PureEffect *); -/* A valid completion consumes read_resource (even on failure); a rejected call - * leaves it untouched. write_reply borrows from the retained ingress. */ -bool pure_runtime_complete(PureTicket, PureResult status, - PureResource *read_resource, PureSpan write_reply, - int64_t revision); -#endif -#endif diff --git a/prototypes/connections/serve.c b/prototypes/connections/serve.c deleted file mode 100644 index 2ea7adb..0000000 --- a/prototypes/connections/serve.c +++ /dev/null @@ -1,1820 +0,0 @@ -#define _GNU_SOURCE -/* pure-serve: the HTTP host for aup modules (docs/research/075). - * - * An app's views and inputs are its HTTP interface, with no routes to write: - * GET /views/NAME answers a view (a number or text as text/plain, a table as - * JSON rows, ?from=N&count=M for a page of them) and POST /inputs/NAME - * delivers an event (a text input's text is the body; a key input is - * pressed, down then up; a message takes its u32 fields from the query and - * its bytes field from the body, and answers what its handler replies, a - * field of the message). A failed event answers with its fault. - * - * A module with keyed messages named show and edit also serves the document - * contract of server.c (GET and PUT /objects/KEY, If-Match or If-None-Match: - * *, ETag = revision, X-Pure-Error): GET is its show message and PUT its edit - * message. HTTP/1.1 connections persist and may pipeline; an epoll thread - * accepts, reads and writes without blocking. Requests go through the - * runtime core (a work slot per connection, the payload budget, ingress - * owned by the core, write replies borrowed from the request). After each - * round of events every ready request is applied to the module with - * deferred saving, one save stores the burst, and only then are its - * requests answered: an answer is durable, and a failed save fails the - * whole burst, reads included. - * - * With --threads N each thread has its own listener on the same port - * (SO_REUSEPORT), epoll set and runtime domain; the kernel spreads - * connections over them. A module of keyed documents alone runs one app per - * thread: they share nothing but the store, whose commits take turns and - * whose revisions settle concurrent edits of a key. States have one owner: - * for a module with them, one more thread owns the app, and the connection - * threads hand it their ready requests and answer what it hands back. Its - * bursts take the requests of every thread, one save each. */ -#include "deliver.h" -#include "message-json.h" -#include "module-store.h" -#include "module-text.h" -#include "runtime.h" -#include "store.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { HEADER_LIMIT = 4096, EVENTS = 256, BATCH = 256 }; -typedef enum { CLOSED, READING_HEAD, READING_BODY, WAITING, WRITING } Phase; -typedef struct Host Host; - -/* What a request names: a document, a view (a table's rows from, count) or - * an input. Kept per work slot, so an accepted event runs even if its - * connection is gone before the burst. */ -typedef enum { ROUTE_NONE, ROUTE_OBJECT, ROUTE_VIEW, ROUTE_INPUT } RouteKind; -typedef struct { RouteKind kind; uint32_t target; size_t from, count; } Route; - -/* Allocated when accepted and reused after closing: memory follows the - * connections that are open, not the configured maximum. */ -typedef struct Connection { - Host *host; - struct Connection *next; /* on the spare or the closed list */ - size_t index; /* in the host's table */ - int fd; - Phase phase; - bool readable, writable, keep_alive, put, post; - uint64_t deadline; - char key[65]; /* the document's key, or the view's or input's name */ - RouteKind route; - /* An app input's message: its fixed fields from the query, which fields - * were given there, and whether the body is a JSON object of the rest. - * Its ingress is [u32le payload size][payload]; with JSON it is [u32le - * payload size | PLACED][u32le start and size of each field][payload: the - * prefix, then the JSON as it came], its strings read where they are. */ - unsigned char fields[8 * PURE_MODULE_FIELDS]; - size_t field_bytes; - bool given[PURE_MODULE_FIELDS], json; - uint32_t input; - unsigned char *ingress; - char source[PURE_SOURCE + 1]; /* a delivered message's sender (X-Pure-Source), or empty */ - uint64_t sequence; /* and its number (X-Pure-Sequence) */ - unsigned batch; /* how many numbered messages the body holds (X-Pure-Count) */ - int64_t expected; - size_t body_size, received, in_used; - unsigned char *body; - PureTicket ticket; - struct iovec *out, small[4]; /* the answer: its head, then its body's extents */ - size_t out_count, out_at; - char head[320]; - char in[HEADER_LIMIT]; /* the header being read, then pipelined bytes */ -} Connection; - -/* One request applied in a burst, for the connection thread that answers - * it. A read answered from the store lends its extents: they point into a - * store mapping, which the resource holds and the runtime charges and - * releases with the ticket. */ -typedef struct { - Host *host; - PureTicket ticket; - PureInput input; - PureResult status; - int64_t revision; - PureResource resource; - PureSpan reply; - struct iovec *extents; - size_t extent_count; - char key[65]; - RouteKind route; - PureFault fault; /* a failed event's */ - const char *type; /* a view's content type */ -} Applied; - -/* The app and its store. A connection thread runs it itself (an app of - * documents only), or one owner thread runs it for all connection threads. - * The owner publishes a snapshot of the states after each save, and the - * connection threads show views from it themselves: reads never wait for - * the owner, which may be applying events or waiting for a sync. */ -typedef struct { - const PureModule *module; - PureStore *store; - PureApp app; - uint32_t show, edit; - uint64_t bursts, burst_events; - Applied *applied; /* a round's requests, grown to fit */ - size_t applied_capacity; - bool wrote; /* the last round saved an event or an edit */ - bool reads_first; /* the owner's: a round's reads before its writes */ - double next[PURE_MODULE_INPUTS]; /* when each timer ticks next (wall clock, ms) */ - uint64_t ticks; - /* The owner's: the last save's snapshot (NULL: the owner shows views), - * taken under the lock, and the app's changes it shows. */ - bool publishes; - pthread_mutex_t published_lock; - PureSnapshot *published; - uint64_t published_changes, publications; - /* The owner's: outputs wired to HTTP receivers, and how far each was trimmed. */ - PureWire *wires[PURE_MODULE_INPUTS]; - unsigned wire_count; - uint64_t trimmed[PURE_MODULE_INPUTS], answers; - bool held[PURE_MODULE_INPUTS]; /* an answer failed to apply: the wire's sends stay until a restart */ -} Executor; - -/* A request made ready: its effect, what it names, and the thread that answers it. */ -typedef struct { Host *host; PureEffect effect; Route route; } Job; - -/* The owner of an app with states served by several connection threads: - * their requests wait here, and it runs them in bursts, one save each - * however many threads they came from, handing each thread back its - * applied requests to answer. */ -typedef struct { - Executor executor; - pthread_mutex_t lock; - pthread_cond_t waiting; - Job *jobs; - size_t count, capacity; - uint64_t arrivals; /* requests handed to the owner, ever */ - bool stopping; - bool delivered; /* a wire delivered a send: trim it */ - Host *hosts; - unsigned host_count; -} Owner; - -struct Host { - const PureModule *module; - bool documents; /* the module serves /objects: it has show and edit */ - Executor local; /* the app, when this thread runs it */ - Owner *owner; /* or the owner that runs it */ - PureApp reader; /* shows the owner's snapshot here */ - Applied *views; /* a round's views shown here, answered together */ - size_t views_capacity; - uint64_t shown; /* views shown from snapshots */ - Route *routes; /* by work slot */ - PureRuntime runtime; - PureWork *work; - Connection **table; /* open connections by index */ - Connection **by_slot; /* the connection waiting on a work slot */ - Connection *spare, *closed; - Job *jobs; /* a round's requests: at most one per work slot */ - size_t capacity, active; - int epoll, listener; - uint64_t timeout_ms; - uint64_t accepted, requests, expired; - /* What the owner hands back to answer (swapped for answering), and the - * eventfd that says so; pending counts requests at the owner. */ - pthread_mutex_t lock; - Applied *done, *answering; - size_t done_count, done_capacity, answering_capacity, pending; - int wake; -}; - -/* Grow an array to hold `need` items; false when memory runs out. */ -static bool grow(void *array, size_t *capacity, size_t need, size_t size) { - if (need <= *capacity) return true; - size_t next = *capacity ? *capacity : 64; - while (next < need) next *= 2; - void *grown = realloc(*(void **)array, next * size); - if (!grown) return false; - *(void **)array = grown; - *capacity = next; - return true; -} - -static atomic_bool stopping; /* lock-free, so a signal handler may set it */ -static void stop(int signal_number) { (void)signal_number; atomic_store(&stopping, true); } - -static uint64_t now_ms(void) { - struct timespec t; - clock_gettime(CLOCK_MONOTONIC, &t); - return (uint64_t)t.tv_sec * 1000 + (uint64_t)t.tv_nsec / 1000000; -} - -/* The time events see: the wall clock, in milliseconds since 1970 (UTC). */ -static double wall_ms(void) { - struct timespec t; - clock_gettime(CLOCK_REALTIME, &t); - return floor((double)t.tv_sec * 1000 + (double)t.tv_nsec / 1e6); -} - -static bool number(const char *text, uint64_t maximum, uint64_t *value) { - if (!*text) return false; - uint64_t n = 0; - for (; *text; ++text) { - if (*text < '0' || *text > '9') return false; - unsigned digit = (unsigned)(*text - '0'); - if (n > (maximum - digit) / 10) return false; - n = n * 10 + digit; - } - *value = n; - return true; -} - -/* ---- Connections ------------------------------------------------------ */ - -static void release_ticket(void *owner) { - Connection *c = owner; - if (c->ticket.runtime && !pure_runtime_release(c->ticket)) abort(); - c->ticket = (PureTicket){0}; - c->body = NULL; -} - -/* Everything an answer borrowed goes back with its ticket: the request's - * payload, a copied reply or the lent store extents. */ -static void finish_output(Connection *c) { - release_ticket(c); - if (c->out != c->small) free(c->out); - c->out = NULL; - c->out_count = c->out_at = 0; -} - -/* Events of this round may still name c: it is reused after the round. */ -static void close_connection(Connection *c) { - if (c->phase == CLOSED) return; - Host *host = c->host; - finish_output(c); - close(c->fd); /* also leaves the epoll set */ - c->phase = CLOSED; - host->table[c->index] = NULL; - c->next = host->closed; - host->closed = c; - --host->active; -} - -/* Answer with a body of `count` extents (`size` bytes in all); the ticket - * keeps them alive. One writev sends as much as the socket takes. */ -static void respond_typed(Connection *c, int status, const char *reason, const char *error, int64_t revision, - const char *type, const struct iovec *extents, size_t count, size_t size) { - char tag[48] = ""; - if (revision > 0) snprintf(tag, sizeof(tag), "ETag: \"%" PRId64 "\"\r\n", revision); - /* A delivery's answer tells how many numbered messages one request may carry. */ - char batch[32] = ""; - if (*c->source) snprintf(batch, sizeof(batch), "X-Pure-Batch: %d\r\n", BATCH); - int length = snprintf(c->head, sizeof(c->head), - "HTTP/1.1 %d %s\r\nContent-Length: %zu\r\n%sContent-Type: %s\r\n" - "Cache-Control: no-store\r\nX-Pure-Error: %s\r\n%s%s\r\n", - status, reason, size, c->keep_alive ? "" : "Connection: close\r\n", type, error, tag, batch); - c->out = count < sizeof(c->small) / sizeof(*c->small) ? c->small : malloc((count + 1) * sizeof(*c->out)); - c->phase = WRITING; - if (!c->out) { c->keep_alive = false; c->out = c->small; count = 0; } /* cannot send it: the close tells */ - c->out[0] = (struct iovec){c->head, (size_t)length}; - if (count) memcpy(c->out + 1, extents, count * sizeof(*extents)); - c->out_count = count + 1; - c->out_at = 0; -} - -static void respond_extents(Connection *c, int status, const char *reason, const char *error, int64_t revision, - const struct iovec *extents, size_t count, size_t size) { - respond_typed(c, status, reason, error, revision, "application/octet-stream", extents, count, size); -} - -static void respond(Connection *c, int status, const char *reason, const char *error, int64_t revision, - PureSpan body) { - struct iovec extent = {(void *)body.data, body.size}; - respond_extents(c, status, reason, error, revision, &extent, body.size ? 1 : 0, body.size); -} - -/* A request refused before it has a ticket; a body it came with is not read. */ -static void reject(Connection *c, int status, const char *reason, const char *error) { - if (c->body_size) c->keep_alive = false; - respond(c, status, reason, error, 0, (PureSpan){0}); -} - -/* A request the host cannot read further: answer and close. */ -static void refuse(Connection *c, int status, const char *reason, const char *error) { - release_ticket(c); - c->keep_alive = false; - respond(c, status, reason, error, 0, (PureSpan){0}); -} - -static void answer(Connection *c, PureOutcome result) { - switch (result.status) { - case PURE_OK: respond(c, 200, "OK", "none", result.revision, result.body); return; - case PURE_CONFLICT: respond(c, 412, "Precondition Failed", "conflict", result.revision, (PureSpan){0}); return; - case PURE_NOT_FOUND: respond(c, 404, "Not Found", "not_found", 0, (PureSpan){0}); break; - case PURE_BUSY: respond(c, 503, "Service Unavailable", "store_busy", 0, (PureSpan){0}); break; - case PURE_BUDGET: respond(c, 503, "Service Unavailable", "payload_budget", 0, (PureSpan){0}); break; - case PURE_ALLOCATION: respond(c, 503, "Service Unavailable", "allocation", 0, (PureSpan){0}); break; - case PURE_CLOSED: respond(c, 503, "Service Unavailable", "stopped", 0, (PureSpan){0}); break; - case PURE_DENIED: respond(c, 403, "Forbidden", "denied", 0, (PureSpan){0}); break; - case PURE_LIMIT: respond(c, 413, "Content Too Large", "limit", 0, (PureSpan){0}); break; - case PURE_UNCERTAIN: respond(c, 500, "Internal Server Error", "durability_unknown", 0, (PureSpan){0}); break; - default: respond(c, 500, "Internal Server Error", "storage", 0, (PureSpan){0}); break; - } - /* A failed request carries no body: the payload can go now. */ - if (!result.body.size) release_ticket(c); -} - -static const char *const fault_names[] = {"none", "divide", "non_finite", "clamp", "limit", "not_found", "encoding", - "exists"}; - -/* An app route's answer: a view's value, or what an event replied. A failed - * event names its fault: 404 not_found, 409 exists, 422 for the others; - * a view without a value is 404 no_value. */ -static void answer_app(Connection *c, PureOutcome result, bool view, PureFault fault, const char *type) { - const char *named = fault != PURE_FAULT_NONE ? fault_names[fault] : NULL; - if (result.status == PURE_OK) { - struct iovec extent = {(void *)result.body.data, result.body.size}; - respond_typed(c, 200, "OK", "none", 0, type ? type : "application/octet-stream", &extent, - result.body.size ? 1 : 0, result.body.size); - } else if (result.status == PURE_NOT_FOUND) { - respond(c, 404, "Not Found", view ? "no_value" : named ? named : "not_found", 0, (PureSpan){0}); - } else if (result.status == PURE_CONFLICT) { - respond(c, 409, "Conflict", named ? named : "conflict", 0, (PureSpan){0}); - } else if (named && (result.status == PURE_LIMIT || result.status == PURE_INVALID)) { - respond(c, 422, "Unprocessable Content", named, 0, (PureSpan){0}); - } else if (result.status == PURE_INVALID) { - respond(c, 400, "Bad Request", "invalid", 0, (PureSpan){0}); /* such as a text that is not UTF-8 */ - } else { - answer(c, result); - return; - } - if (!result.body.size) release_ticket(c); -} - -/* Parse one request head of `size` bytes (through the blank line): the same - * bounded HTTP/1.1 subset as server.c, plus Connection, POST and the app - * routes. A query stays in head for start_request; an unknown path is - * answered 404 once its head is read. */ -static bool parse_head(Connection *c, char *head, size_t size, char **query) { - for (size_t i = 0; i < size; ++i) { - unsigned char b = (unsigned char)head[i]; - if ((b < 32 && b != '\r' && b != '\n' && b != '\t') || b == 127) return false; - } - head[size - 2] = 0; /* keep the last CRLF of the last header */ - char *line = head, *next = strstr(line, "\r\n"); - if (!next) return false; - *next = 0; - char *path = strchr(line, ' '); - if (!path) return false; - *path++ = 0; - c->put = !strcmp(line, "PUT"); - c->post = !strcmp(line, "POST"); - if (!c->put && !c->post && strcmp(line, "GET")) return false; - char *version = strchr(path, ' '); - if (!version) return false; - *version++ = 0; - if (strcmp(version, "HTTP/1.1")) return false; - if ((*query = strchr(path, '?'))) *(*query)++ = 0; - static const struct { const char *prefix; RouteKind kind; } routes[] = { - {"/objects/", ROUTE_OBJECT}, {"/views/", ROUTE_VIEW}, {"/inputs/", ROUTE_INPUT}}; - const char *key = ""; - c->route = ROUTE_NONE; - for (size_t i = 0; i < sizeof(routes) / sizeof(*routes) && c->route == ROUTE_NONE; ++i) - if (!strncmp(path, routes[i].prefix, strlen(routes[i].prefix))) { - c->route = routes[i].kind; - key = path + strlen(routes[i].prefix); - } - size_t key_size = strlen(key); - if (c->route != ROUTE_NONE && (!key_size || key_size > 64)) return false; - for (size_t i = 0; i < key_size; ++i) { - char b = key[i]; - if (!((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '-' || b == '_')) - return false; - } - memcpy(c->key, key, key_size + 1); - /* Each route has its method: documents GET and PUT, views GET, inputs POST. */ - if ((c->route == ROUTE_OBJECT && (c->post || *query)) || (c->route == ROUTE_VIEW && (c->put || c->post)) || - (c->route == ROUTE_INPUT && !c->post)) return false; - bool length_seen = false, condition_seen = false, host_seen = false; - c->source[0] = 0; - c->sequence = 0; - c->batch = 1; - bool counted = false; - c->body_size = 0; - c->json = false; - c->expected = -1; - c->keep_alive = true; - line = next + 2; - while (*line) { - next = strstr(line, "\r\n"); - if (!next) return false; - *next = 0; - char *value = strchr(line, ':'); - if (!value || value == line) return false; - *value++ = 0; - for (char *p = line; *p; ++p) - if (!((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || (*p >= '0' && *p <= '9') || *p == '-')) - return false; - while (*value == ' ' || *value == '\t') ++value; - char *tail = value + strlen(value); - while (tail > value && (tail[-1] == ' ' || tail[-1] == '\t')) *--tail = 0; - uint64_t n; - if (!strcasecmp(line, "Content-Length")) { - if (length_seen || !number(value, SIZE_MAX, &n)) return false; - length_seen = true; - c->body_size = (size_t)n; - } else if (!strcasecmp(line, "Host")) { - if (host_seen || !*value) return false; - host_seen = true; - } else if (!strcasecmp(line, "If-Match")) { - if (condition_seen || tail - value < 3 || *value != '"' || tail[-1] != '"') return false; - tail[-1] = 0; - if (!number(value + 1, INT64_MAX, &n) || !n) return false; - c->expected = (int64_t)n; - condition_seen = true; - } else if (!strcasecmp(line, "If-None-Match")) { - if (condition_seen || strcmp(value, "*")) return false; - c->expected = 0; - condition_seen = true; - } else if (!strcasecmp(line, "Content-Type")) { - c->json = !strncasecmp(value, "application/json", 16); - } else if (!strcasecmp(line, "Connection")) { - if (!strcasecmp(value, "close")) c->keep_alive = false; - else if (strcasecmp(value, "keep-alive")) return false; - } else if (!strcasecmp(line, "X-Pure-Source")) { - /* A message delivered from another app's output: applied once per number. */ - size_t length = strlen(value); - if (*c->source || !length || length > PURE_SOURCE) return false; - for (size_t i = 0; i < length; ++i) if (value[i] <= ' ' || value[i] > '~') return false; - memcpy(c->source, value, length + 1); - } else if (!strcasecmp(line, "X-Pure-Sequence")) { - if (c->sequence || !number(value, UINT64_MAX, &n) || !n) return false; - c->sequence = n; - } else if (!strcasecmp(line, "X-Pure-Count")) { - /* Numbered messages from X-Pure-Sequence on, as a JSON array. */ - if (counted || !number(value, BATCH, &n) || !n) return false; - c->batch = (unsigned)n; - counted = true; - } else if (!strcasecmp(line, "Transfer-Encoding") || !strcasecmp(line, "Expect")) return false; - line = next + 2; - } - if (!*c->source != !c->sequence || (*c->source && c->route != ROUTE_INPUT) || (counted && !*c->source) || - c->sequence > UINT64_MAX - c->batch) return false; - if (c->put) return host_seen && length_seen && condition_seen; - if (c->post) return host_seen && !condition_seen; /* no Content-Length: no body */ - return host_seen && !condition_seen && c->body_size == 0; -} - -static void put_u32(unsigned char *p, uint32_t v) { for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(v >> (8 * i)); } -static void put_u64(unsigned char *p, uint64_t v) { for (unsigned i = 0; i < 8; ++i) p[i] = (unsigned char)(v >> (8 * i)); } -static uint32_t get_u32(const unsigned char *p) { return p[0] | p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; } -static uint64_t get_u64(const unsigned char *p) { return get_u32(p) | (uint64_t)get_u32(p + 4) << 32; } - -/* What an app route names in the module, with a message's fixed fields - * taken from the query into c->fields. NULL, or the error to answer (404 - * for a name the module lacks, 400 otherwise). */ -static const char *resolve(const Host *host, Connection *c, char *query, Route *route, int *status) { - const PureModule *m = host->module; - *status = 404; - c->field_bytes = 0; - memset(c->given, 0, sizeof(c->given)); - if (route->kind == ROUTE_NONE || (route->kind == ROUTE_OBJECT && !host->documents)) return "route"; - if (route->kind == ROUTE_OBJECT) return NULL; - int found = route->kind == ROUTE_VIEW ? pure_module_view(m, c->key) : pure_module_input(m, c->key); - /* An output is the app's to send and a timer ticks by itself: neither is posted. */ - if (found >= 0 && route->kind == ROUTE_INPUT && - (m->input[found].kind == PURE_KIND_OUTPUT || m->input[found].kind == PURE_KIND_TIMER)) found = -1; - if (found < 0) return route->kind == ROUTE_VIEW ? "no_view" : "no_input"; - *status = 400; - route->target = (uint32_t)found; - route->from = 0; - route->count = SIZE_MAX; - const PureInputShape *shape = route->kind == ROUTE_INPUT ? &m->input[found] : NULL; - bool message = shape && shape->kind == PURE_KIND_MESSAGE; - if (message && shape->keyed) return "keyed"; - if (*c->source && !message) return "delivery"; - if (!host->owner) c->source[0] = 0; /* an app without states or outputs applies it as a message */ - if (shape) { - /* A body is a text input's text, a message's one variable field, or a - * JSON object of a message's fields (two variable fields need one). */ - unsigned variables = 0; - for (uint32_t f = 0; message && f < shape->field_count; ++f) - variables += m->field[shape->field_start + f].type == PURE_FIELD_BYTES || - m->field[shape->field_start + f].type == PURE_FIELD_TEXT; - if (c->json && !message) return "json"; - if (!c->json && (variables > 1 || (c->body_size && shape->kind == PURE_KIND_KEY) || - (c->body_size && message && !variables))) return variables > 1 ? "json" : "body"; - c->input = (uint32_t)found; - c->field_bytes = message ? shape->prefix : 0; - } - /* name=value pairs: a table's from and count, a message's fixed fields. */ - for (char *pair = query; pair && *pair;) { - char *end = strchr(pair, '&'), *value = strchr(pair, '='); - if (end) *end = 0; - if (!value || (end && value > end)) return "query"; - *value++ = 0; - if (route->kind == ROUTE_VIEW) { - uint64_t n; - bool from = !strcmp(pair, "from"); - if (!number(value, SIZE_MAX, &n) || m->view_type[found] != PURE_VIEW_TABLE || (!from && strcmp(pair, "count"))) - return "query"; - if (from) route->from = (size_t)n; - else route->count = (size_t)n; - } else { - uint32_t f = 0; - while (f < (message ? shape->field_count : 0) && strcmp(m->field[shape->field_start + f].name, pair)) ++f; - const PureField *field = message && f < shape->field_count ? &m->field[shape->field_start + f] : NULL; - if (!field || c->given[f] || (field->type != PURE_FIELD_U32 && field->type != PURE_FIELD_NUMBER) || - !pure_json_fixed(field, (const unsigned char *)value, strlen(value), c->fields)) return "query"; - c->given[f] = true; - } - pair = end ? end + 1 : NULL; - } - for (uint32_t f = 0; message && !c->json && f < shape->field_count; ++f) { - uint8_t type = m->field[shape->field_start + f].type; - if ((type == PURE_FIELD_U32 || type == PURE_FIELD_NUMBER) && !c->given[f]) return "missing_field"; - } - for (uint32_t f = 0; c->batch > 1 && f < shape->field_count; ++f) - if (c->given[f]) return "delivery"; /* a batch's fields are all in its objects */ - if (c->batch > 1 && !c->json) return "delivery"; - if (!*c->source) c->batch = 1; - return NULL; -} - -/* Parsing a JSON body in place: the payload it is in, the placed fields' - * table, and which fields it gave. */ -#define PLACED (UINT32_C(1) << 31) -/* After the size: a delivered message's number (u64le), how many follow it - * (u32le: a batch) and its source (u8 length and the bytes), in DELIVERY - * bytes before the tables; then a table for each message, a prefix for each - * and the JSON. */ -#define DELIVERED (UINT32_C(1) << 30) -enum { DELIVERY = 80 }; -/* An app input's JSON body becomes its message: an object with a member for - * each field the query did not give (a u32, a number, or a string for bytes - * and text), groups as objects inside it, and nothing else. Fixed fields go - * into the prefix; strings stay in the body, and the table says where. - * NULL, or the error to answer. */ -static const char *convert(Connection *c) { - const PureInputShape *shape = &c->host->module->input[c->input]; - uint32_t delivered = get_u32(c->ingress) & DELIVERED; - size_t table = 8 * (size_t)shape->field_count; - unsigned char *tables = c->ingress + 4 + (delivered ? DELIVERY : 0), *prefixes = tables + c->batch * table; - unsigned char *json = c->body; - size_t size = c->body_size, at = pure_json_space(json, 0, size); - /* A batch is an array of its messages' objects. */ - if (c->batch > 1) { - if (at == size || json[at++] != '[') return "json"; - at = pure_json_space(json, at, size); - } - for (unsigned i = 0; i < c->batch; ++i) { - if (i) { - if (at == size || json[at++] != ',') return "json"; - at = pure_json_space(json, at, size); - } - PureJsonMessage b = {.module = c->host->module, .shape = shape, .table = tables + i * table, - .payload = prefixes + i * c->field_bytes, .given = c->given}; - const char *why = pure_json_message(&b, json, size, &at); - if (why) return why; - at = pure_json_space(json, at, size); - for (uint32_t f = 0; f < shape->field_count; ++f) - if (!b.seen[f] && !c->given[f]) return "missing_field"; - } - if (c->batch > 1 && (at == size || json[at++] != ']')) return "json"; - if (pure_json_space(json, at, size) != size) return "json"; - put_u32(c->ingress, (uint32_t)(c->batch * c->field_bytes + c->body_size) | PLACED | delivered); - return NULL; -} - -/* A request's body is in: an app input's JSON becomes its payload, and the - * request is ready, or it is answered 400 and never runs. */ -static void body_done(Connection *c) { - const char *why = c->route == ROUTE_INPUT && c->json ? convert(c) : NULL; - if (why) { - release_ticket(c); /* never made ready: it never runs */ - respond(c, 400, "Bad Request", why, 0, (PureSpan){0}); - return; - } - if (!pure_runtime_ready(c->ticket)) abort(); - c->phase = WAITING; -} - -/* A whole head is in `in`: admit the request and take any body bytes that - * came with it; later bytes stay for the next request. */ -static void start_request(Connection *c, size_t head_size) { - Host *host = c->host; - char head[HEADER_LIMIT], *query = NULL; - memcpy(head, c->in, head_size); - size_t extra = c->in_used - head_size; - memmove(c->in, c->in + head_size, extra); - c->in_used = extra; - if (!parse_head(c, head, head_size, &query)) { refuse(c, 400, "Bad Request", "request"); return; } - if (c->body_size > PURE_OBJECT_LIMIT) { refuse(c, 413, "Content Too Large", "limit"); return; } - ++host->requests; - Route route = {.kind = c->route}; - int status; - const char *why = resolve(host, c, query, &route, &status); - if (why) { reject(c, status, status == 404 ? "Not Found" : "Bad Request", why); return; } - /* A document's PUT carries its body; an input's event its payload's size, - * the payload (the fixed fields, then the body), and room for its JSON. */ - bool write = route.kind == ROUTE_INPUT || c->put; - size_t table = route.kind == ROUTE_INPUT && c->json ? 8 * (size_t)host->module->input[c->input].field_count : 0; - size_t lead = 4 + (route.kind == ROUTE_INPUT && *c->source ? DELIVERY : 0); - /* A batch has a table and a prefix for each of its messages. */ - size_t input_size = route.kind == ROUTE_INPUT ? lead + c->batch * (table + c->field_bytes) + c->body_size : - write ? c->body_size : 0; - PureResult admitted = pure_runtime_accept(&host->runtime, write ? PURE_INPUT_WRITE : PURE_INPUT_READ, c->key, - route.kind == ROUTE_INPUT ? 0 : c->expected, input_size, &c->ticket); - if (admitted != PURE_OK) { - c->ticket = (PureTicket){0}; - /* An unread body would desynchronize the stream. */ - if (c->body_size) c->keep_alive = false; - answer(c, (PureOutcome){.status = admitted}); - return; - } - host->by_slot[c->ticket.slot] = c; - host->routes[c->ticket.slot] = route; - if (!write) { c->phase = WAITING; return; } - unsigned char *ingress = c->ingress = pure_runtime_ingress(c->ticket, NULL); /* NULL for no bytes */ - c->body = ingress; - if (route.kind == ROUTE_INPUT) { - put_u32(ingress, (uint32_t)(c->batch * c->field_bytes + c->body_size) | (lead > 4 ? DELIVERED : 0)); - if (lead > 4) { - size_t length = strlen(c->source); - put_u64(ingress + 4, c->sequence); - put_u32(ingress + 12, c->batch); - ingress[16] = (unsigned char)length; - memcpy(ingress + 17, c->source, length); - } - if (c->field_bytes) memcpy(ingress + lead + c->batch * table, c->fields, c->field_bytes); - c->body = ingress + lead + c->batch * (table + c->field_bytes); - } - size_t take = c->in_used < c->body_size ? c->in_used : c->body_size; - if (take) memcpy(c->body, c->in, take); - memmove(c->in, c->in + take, c->in_used - take); - c->in_used -= take; - c->received = take; - c->phase = READING_BODY; - if (c->received == c->body_size) body_done(c); -} - -static size_t head_end(const Connection *c) { - for (size_t i = 3; i < c->in_used; ++i) - if (!memcmp(c->in + i - 3, "\r\n\r\n", 4)) return i + 1; - return 0; -} - -/* Advance a connection as far as its socket allows without blocking. */ -static void drive(Connection *c) { - for (;;) { - if (c->phase == READING_HEAD) { - size_t end = head_end(c); - if (end) { start_request(c, end); continue; } - if (c->in_used == HEADER_LIMIT) { refuse(c, 431, "Request Header Fields Too Large", "headers"); continue; } - if (!c->readable) return; - ssize_t n = recv(c->fd, c->in + c->in_used, HEADER_LIMIT - c->in_used, 0); - if (n > 0) { c->in_used += (size_t)n; c->deadline = now_ms() + c->host->timeout_ms; continue; } - if (n < 0 && errno == EAGAIN) { c->readable = false; return; } - if (n < 0 && errno == EINTR) continue; - close_connection(c); - return; - } - if (c->phase == READING_BODY) { - if (!c->readable) return; - ssize_t n = recv(c->fd, c->body + c->received, c->body_size - c->received, 0); - if (n > 0) { - c->received += (size_t)n; - c->deadline = now_ms() + c->host->timeout_ms; - if (c->received == c->body_size) body_done(c); - continue; - } - if (n < 0 && errno == EAGAIN) { c->readable = false; return; } - if (n < 0 && errno == EINTR) continue; - close_connection(c); - return; - } - if (c->phase == WRITING) { - if (!c->writable) return; - size_t left = c->out_count - c->out_at; - ssize_t n = writev(c->fd, c->out + c->out_at, (int)(left < IOV_MAX ? left : IOV_MAX)); - if (n < 0 && errno == EAGAIN) { c->writable = false; return; } - if (n < 0 && errno == EINTR) continue; - if (n < 0) { close_connection(c); return; } - c->deadline = now_ms() + c->host->timeout_ms; /* a slow reader is fine while it reads */ - for (size_t sent = (size_t)n; c->out_at < c->out_count;) { - struct iovec *v = &c->out[c->out_at]; - if (sent < v->iov_len) { - v->iov_base = (char *)v->iov_base + sent; - v->iov_len -= sent; - break; - } - sent -= v->iov_len; - ++c->out_at; - } - if (c->out_at < c->out_count) continue; - /* Answered: the next request may already be here. */ - finish_output(c); - c->deadline = now_ms() + c->host->timeout_ms; - if (!c->keep_alive) { close_connection(c); return; } - c->phase = READING_HEAD; - continue; - } - return; /* WAITING: the next burst answers */ - } -} - -/* ---- Bursts ----------------------------------------------------------- */ - -static void release_copy(void *allocation, size_t size) { - (void)size; - free(allocation); -} - -static void release_loan(void *allocation, size_t size) { - (void)size; - pure_mapping_release(allocation); -} - -static bool allocate_ingress(void *context, size_t size, PureResource *out) { - (void)context; - void *bytes = malloc(size ? size : 1); /* a key press or an empty text has no payload */ - if (!bytes) return false; - *out = (PureResource){.body = {bytes, size}, .allocation = bytes, .allocation_size = size, .release = release_copy}; - return true; -} - -typedef struct { unsigned char *bytes; size_t size; const unsigned char *first; size_t spans; } Reply; - -static bool measure_reply(void *context, PureSpan span) { - Reply *r = context; - if (!r->spans++) r->first = span.data; - r->size += span.size; - return true; -} - -/* A value's extents, if the store mapping holds every one. */ -typedef struct { struct iovec *extents; size_t count, capacity; const PureMapping *mapping; bool held; } Gather; - -static bool gather_extents(void *context, PureSpan span) { - Gather *g = context; - if (!span.size) return true; - g->held = g->held && pure_mapping_holds(g->mapping, span) && - grow(&g->extents, &g->capacity, g->count + 1, sizeof(*g->extents)); - if (g->held) g->extents[g->count++] = (struct iovec){(void *)span.data, span.size}; - return g->held; -} - -/* A gathered value goes out as the mapping's extents, not copied: the answer - * holds the mapping (charged to the budget) until it is sent and its ticket - * released. false (and nothing held) when some extent was not stored. */ -static bool lend(Applied *out, PureMapping *mapping, Gather *gather, bool read, size_t size) { - if (!read || !gather->held) { - free(gather->extents); - pure_mapping_release(mapping); - return false; - } - out->extents = gather->extents; - out->extent_count = gather->count; - out->resource = (PureResource){.body = {NULL, size}, .allocation = mapping, .allocation_size = size, - .release = release_loan}; - return true; -} - -static bool copy_reply(void *context, PureSpan span) { - Reply *r = context; - if (span.size) memcpy(r->bytes + r->size, span.data, span.size); - r->size += span.size; - return true; -} - -/* ---- App routes -------------------------------------------------------- */ - -typedef struct { unsigned char *bytes; size_t size, capacity; bool failed; } Output; - -static bool put_bytes(Output *o, const void *bytes, size_t size) { - if (o->failed) return false; - if (size > o->capacity - o->size) { - size_t capacity = o->capacity ? o->capacity : 256; - while (capacity - o->size < size) { - if (capacity > SIZE_MAX / 2) { o->failed = true; return false; } - capacity *= 2; - } - unsigned char *grown = realloc(o->bytes, capacity); - if (!grown) { o->failed = true; return false; } - o->bytes = grown; - o->capacity = capacity; - } - if (size) memcpy(o->bytes + o->size, bytes, size); - o->size += size; - return true; -} - -static bool put_span(void *context, PureSpan span) { return put_bytes(context, span.data, span.size); } - -/* A JSON string's contents: quotes, backslashes and control bytes escaped, - * other bytes (UTF-8, which Pure keeps valid) as they are. */ -static bool put_json_span(void *context, PureSpan span) { - Output *o = context; - const unsigned char *p = span.data; - size_t plain = 0; - for (size_t i = 0; i < span.size; ++i) { - unsigned char b = p[i]; - if (b >= 0x20 && b != '"' && b != '\\') continue; - char escape[8]; - int length = b == '"' || b == '\\' ? snprintf(escape, sizeof(escape), "\\%c", b) : - snprintf(escape, sizeof(escape), "\\u%04x", b); - put_bytes(o, p + plain, i - plain); - put_bytes(o, escape, (size_t)length); - plain = i + 1; - } - return put_bytes(o, p + plain, span.size - plain); -} - -static void put_number(Output *o, double value) { - char digits[32]; - put_bytes(o, digits, pure_format_number(value, digits)); -} - -/* A view's value: a number in Pure's shortest form or a text, as text; a - * table as {"rows":N,"from":F,"items":[{column: value}...]} for the rows - * asked, each cell computed only for them (a cell without a value is null). */ -static PureResult show_view(PureApp *app, const Route *route, size_t available, Applied *answer) { - const PureModule *m = app->module; - uint32_t view = route->target; - PureResource *out = &answer->resource; - const char **type = &answer->type; - /* Room for a page of short rows at once. */ - Output o = {0}; - PureResult status = PURE_OK; - double value; - if (m->view_type[view] == PURE_VIEW_TABLE && !(o.bytes = malloc(o.capacity = 1024))) return PURE_ALLOCATION; - if (m->view_type[view] == PURE_VIEW_NUM) { - *type = "text/plain; charset=utf-8"; - if (pure_app_view(app, view, &value)) put_number(&o, value); - else status = PURE_NOT_FOUND; - } else if (m->view_type[view] == PURE_VIEW_TEXT) { - *type = "text/plain; charset=utf-8"; - /* A text state shown as it is goes out as its stored extents. */ - int state = pure_module_text_state(m, view); - PureDraft *text = state >= 0 ? app->texts[state] : NULL; - PureVersion version = text ? pure_draft_version(text) : (PureVersion){0, 0}; - if (version.bytes > available) return PURE_BUDGET; - PureMapping *mapping = version.bytes ? pure_draft_mapping(text) : NULL; - if (mapping) { - Gather gather = {NULL, 0, 0, mapping, true}; - bool read = pure_draft_read_version(text, version, 0, (size_t)version.bytes, gather_extents, &gather) == PURE_OK; - if (lend(answer, mapping, &gather, read, (size_t)version.bytes)) return PURE_OK; - } - status = pure_app_view_text(app, view, put_span, &o); - } else { - *type = "application/json"; - size_t rows = pure_app_rows(app, view), from = route->from < rows ? route->from : rows; - size_t end = from + (route->count < rows - from ? route->count : rows - from); - /* Each column's key, ,"name": (the first row's first without the comma), made once. */ - struct { uint32_t view; size_t length; char key[PURE_MODULE_NAME + 4]; } columns[PURE_MODULE_VIEWS]; - unsigned count = 0; - for (uint32_t column = 0; column < m->views; ++column) - if (m->view_table[column] == view + 1) { - columns[count].view = column; - columns[count].length = (size_t)snprintf(columns[count].key, sizeof(columns[count].key), ",\"%s\":", - m->view_names[column]); - ++count; - } - char head[80]; - put_bytes(&o, head, (size_t)snprintf(head, sizeof(head), "{\"rows\":%zu,\"from\":%zu,\"items\":[", rows, from)); - for (size_t row = from; row < end && !o.failed; ++row) { - put_bytes(&o, ",{" + (row == from), 2 - (row == from)); - for (unsigned k = 0; k < count; ++k) { - put_bytes(&o, columns[k].key + !k, columns[k].length - !k); - if (m->view_type[columns[k].view] == PURE_VIEW_NUM) { - if (pure_app_cell(app, columns[k].view, row, &value)) put_number(&o, value); - else put_bytes(&o, "null", 4); - continue; - } - size_t mark = o.size; - put_bytes(&o, "\"", 1); - if (pure_app_cell_text(app, columns[k].view, row, put_json_span, &o) == PURE_OK) put_bytes(&o, "\"", 1); - else if (!o.failed) { - o.size = mark; - put_bytes(&o, "null", 4); - } - } - put_bytes(&o, "}", 1); - } - put_bytes(&o, "]}", 2); - } - if (status == PURE_OK && !o.bytes && !(o.bytes = malloc(1))) o.failed = true; /* an empty value has a buffer too */ - if (status == PURE_OK && o.failed) status = PURE_ALLOCATION; - if (status == PURE_OK && o.size > available) status = PURE_BUDGET; - if (status != PURE_OK) { - free(o.bytes); - return status; - } - *out = (PureResource){.body = {o.bytes, o.size}, .allocation = o.bytes, .allocation_size = o.size, - .release = release_copy}; - return PURE_OK; -} - -/* An input's event: a text, a key pressed (down, then up), or a message - * whose reply (a field of it) is borrowed from the request. */ -static PureResult apply_input(PureApp *app, const Route *route, const PureEffect *effect, Applied *out) { - const PureInputShape *shape = &app->module->input[route->target]; - uint32_t header = get_u32(effect->value.data); - const unsigned char *delivery = effect->value.data + 4; - unsigned batch = header & DELIVERED ? get_u32(delivery + 8) : 1; - size_t lead = 4 + (header & DELIVERED ? DELIVERY : 0), table = header & PLACED ? 8 * (size_t)shape->field_count : 0; - PureSpan payload = {effect->value.data + lead + batch * table, header & ~(PLACED | DELIVERED)}; - PureResult status; - if (shape->kind == PURE_KIND_TEXT) { - status = pure_app_text(app, route->target, payload); - } else if (shape->kind == PURE_KIND_KEY) { - status = pure_app_input(app, route->target, PURE_EDGE_DOWN); - if (status == PURE_OK) status = pure_app_input(app, route->target, PURE_EDGE_UP); - } else { - uint32_t placed[2 * PURE_MODULE_FIELDS]; - for (size_t i = 0; table && i < 2 * (size_t)shape->field_count; ++i) - placed[i] = get_u32(effect->value.data + lead + 4 * i); - if (header & DELIVERED) { - char source[PURE_SOURCE + 1]; - memcpy(source, delivery + 13, delivery[12]); - source[delivery[12]] = 0; - status = pure_app_deliver(app, route->target, source, get_u64(delivery), payload, table ? placed : NULL); - /* The rest of a batch, in order: an app's refusal was heard and the - * batch goes on; another failure stops it, to be sent again. */ - for (unsigned i = 1; i < batch && (status == PURE_OK || app->fault != PURE_FAULT_NONE); ++i) { - const unsigned char *at = effect->value.data + lead + i * table; - for (size_t k = 0; k < 2 * (size_t)shape->field_count; ++k) placed[k] = get_u32(at + 4 * k); - PureSpan next = {payload.data + i * shape->prefix, payload.size - i * shape->prefix}; - status = pure_app_deliver(app, route->target, source, get_u64(delivery) + i, next, placed); - } - if (batch > 1 && app->fault != PURE_FAULT_NONE) status = PURE_OK; - } else - status = table ? pure_app_message_fields(app, route->target, NULL, -1, payload, placed) : - pure_app_message(app, route->target, NULL, -1, payload); - Reply reply = {0}; - if (status == PURE_OK) pure_app_reply(app, measure_reply, &reply); - out->reply = (PureSpan){reply.size ? reply.first : NULL, reply.size}; - } - out->fault = app->fault; - return status; -} - -static void apply_effect(Executor *ex, const Job *job, Applied *out) { - PureApp *app = &ex->app; - const PureEffect *effect = &job->effect; - app->now = wall_ms(); - *out = (Applied){.host = job->host, .ticket = effect->ticket, .input = effect->input, .route = job->route.kind}; - memcpy(out->key, effect->key, strlen(effect->key) + 1); - if (job->route.kind == ROUTE_VIEW) { - out->status = show_view(app, &job->route, effect->available, out); - return; - } - if (job->route.kind == ROUTE_INPUT) { - out->status = apply_input(app, &job->route, effect, out); - if (out->status == PURE_OK) ++ex->burst_events; - return; - } - bool read = effect->input == PURE_INPUT_READ; - out->status = pure_app_message(app, read ? ex->show : ex->edit, effect->key, - read ? -1 : effect->expected, read ? (PureSpan){0} : effect->value); - out->revision = app->revision; - if (out->status != PURE_OK) return; - ++ex->burst_events; - Reply reply = {0}; - pure_app_reply(app, measure_reply, &reply); - if (!read) { - /* Checked at start: an edit replies a field of its request. */ - out->reply = (PureSpan){reply.size ? reply.first : NULL, reply.size}; - return; - } - if (reply.size > effect->available) { out->status = PURE_BUDGET; return; } - if (!reply.size) return; - /* A stored document is lent, extent by extent, not copied. */ - PureMapping *mapping = app->document ? pure_draft_mapping(app->document) : NULL; - if (mapping) { - Gather gather = {NULL, 0, 0, mapping, true}; - bool read = pure_app_reply(app, gather_extents, &gather) == PURE_OK; - if (lend(out, mapping, &gather, read, reply.size)) return; - } - unsigned char *copy = malloc(reply.size); - if (!copy) { out->status = PURE_ALLOCATION; return; } - reply = (Reply){.bytes = copy}; - pure_app_reply(app, copy_reply, &reply); - out->resource = (PureResource){.body = {copy, reply.size}, .allocation = copy, .allocation_size = reply.size, - .release = release_copy}; -} - -/* After a save, before its answers go: from now on views show what it - * saved, so a read made after a write's answer sees the write. A failed save - * returns the app to the last save's states, which the snapshot still shows. */ -static void publish(Executor *ex) { - if (!ex->publishes || (ex->published && ex->published_changes == ex->app.changes)) return; - PureSnapshot *made = pure_app_snapshot(&ex->app), *old; - pthread_mutex_lock(&ex->published_lock); - old = ex->published; - ex->published = made; - pthread_mutex_unlock(&ex->published_lock); - pure_snapshot_release(old); - ex->published_changes = ex->app.changes; - ++ex->publications; -} - -/* The saved sends of wired outputs not queued yet go to their wires, in - * order; a wire that cannot take one takes the rest at a later save. The - * outbox is looked at newest first, only as far back as the new sends go, - * however many older ones wait for their receivers. */ -typedef struct { uint32_t wire; uint64_t seq; PureSpan payload; } Fresh; -typedef struct { Executor *executor; Fresh *fresh; size_t count, capacity; unsigned done; bool failed; } Queueing; - -static bool find_fresh(void *context, uint32_t output, uint64_t seq, PureSpan payload) { - Queueing *q = context; - Executor *ex = q->executor; - for (unsigned i = 0; i < ex->wire_count; ++i) { - if (pure_wire_output(ex->wires[i]) != output || (q->done >> i & 1)) continue; - if (seq <= pure_wire_queued(ex->wires[i])) { - q->done |= 1u << i; /* older ones were queued before */ - break; - } - if (q->count == q->capacity) { - size_t capacity = q->capacity ? 2 * q->capacity : 64; - Fresh *grown = realloc(q->fresh, capacity * sizeof(*grown)); - if (!grown) { q->failed = true; return false; } - q->fresh = grown; - q->capacity = capacity; - } - q->fresh[q->count++] = (Fresh){i, seq, payload}; - break; - } - return q->done != (ex->wire_count >= 32 ? UINT32_MAX : (1u << ex->wire_count) - 1); -} - -static void after_save(Executor *ex) { - publish(ex); - if (!ex->wire_count) return; - Queueing q = {.executor = ex}; - pure_app_outbox_newest(&ex->app, find_fresh, &q); - bool blocked[PURE_MODULE_INPUTS] = {false}; - for (size_t k = q.failed ? 0 : q.count; k-- > 0;) { - const Fresh *f = &q.fresh[k]; - if (!blocked[f->wire]) blocked[f->wire] = !pure_wire_queue(ex->wires[f->wire], f->seq, f->payload); - } - free(q.fresh); -} - -/* The answers the wires made become events, in order; then what the wires - * delivered leaves the outbox (with the next save). True when an answer was - * applied (the caller saves). */ -static bool take_deliveries(Executor *ex) { - bool answered = false; - for (unsigned i = 0; i < ex->wire_count; ++i) { - PureWireAnswer answer; - while (pure_wire_answer(ex->wires[i], &answer)) { - ex->app.now = wall_ms(); - PureResult result = pure_app_deliver(&ex->app, answer.input, answer.source, answer.seq, answer.payload, - answer.placed); - if (result != PURE_OK && ex->app.fault == PURE_FAULT_NONE) { - ex->held[i] = true; /* not applied and not refused: sent again after a restart */ - fprintf(stderr, "{\"wire\":\"%s\",\"seq\":%" PRIu64 ",\"error\":\"its answer was not applied\"}\n", - answer.source, answer.seq); - } - pure_app_finish(&ex->app); - answered = true; - ++ex->answers; - } - uint64_t delivered = pure_wire_delivered(ex->wires[i]); - if (delivered <= ex->trimmed[i] || ex->held[i]) continue; - pure_app_trim(&ex->app, pure_wire_output(ex->wires[i]), delivered); - ex->trimmed[i] = delivered; - } - return answered; -} - -/* One save for a burst: its requests are durable with it or fail with it. */ -static void save_burst(Executor *ex, Applied *applied, size_t count) { - if (!count) return; - for (size_t i = 0; i < count; ++i) - ex->wrote = ex->wrote || (applied[i].status == PURE_OK && applied[i].input == PURE_INPUT_WRITE); - PureResult saved = pure_app_save(&ex->app); - ++ex->bursts; - if (saved == PURE_OK) after_save(ex); - for (size_t i = 0; i < count; ++i) - if (applied[i].status == PURE_OK) applied[i].status = saved; -} - -/* Run a round's requests in bursts. A burst ends before a document's key - * repeats or before its documents would outgrow one save, so every request - * is stored by its burst's save or fails with it; views and events of the - * app's states join any burst. Each group of answers goes to finish. - * - * The owner takes a round's reads first and answers them at once: a - * round's requests come from different connections (each has one request - * at a time), so nothing orders them, and placed before the writes the - * reads see only saved states (a round starts from a save). With writes - * going on this halves the reads' wait and doubles both rates. A single - * thread does not: it waits out each save itself, and reads answered first - * come back first, splitting its next writes over more saves (the owner - * gathers after a save instead). */ -static void execute(Executor *ex, const Job *jobs, size_t count, void (*finish)(void *, Applied *, size_t), - void *context) { - if (!count) return; - if (!grow(&ex->applied, &ex->applied_capacity, count, sizeof(*ex->applied))) abort(); - Applied *applied = ex->applied; - /* One look at the store per round: a request made after another shard's - * answer was sent is read after that answer's commit. */ - bool pinned = pure_store_pin(ex->store, true) == PURE_OK; - size_t reads = 0; - for (size_t i = 0; ex->reads_first && i < count; ++i) - if (jobs[i].effect.input == PURE_INPUT_READ) apply_effect(ex, &jobs[i], &applied[reads++]); - if (reads) finish(context, applied, reads); - size_t n = reads, start = reads, keys = 0; - for (size_t i = 0; i < count; ++i) { - if (ex->reads_first && jobs[i].effect.input == PURE_INPUT_READ) continue; - bool document = jobs[i].route.kind == ROUTE_OBJECT, repeated = false; - for (size_t k = start; k < n && document && !repeated; ++k) - repeated = applied[k].route == ROUTE_OBJECT && !strcmp(applied[k].key, jobs[i].effect.key); - if (repeated || (document && keys == PURE_APP_PENDING)) { - save_burst(ex, applied + start, n - start); - finish(context, applied + start, n - start); - start = n; - keys = 0; - } - apply_effect(ex, &jobs[i], &applied[n++]); - keys += document; - } - save_burst(ex, applied + start, n - start); - finish(context, applied + start, n - start); - if (pinned) pure_store_pin(ex->store, false); -} - -/* Tick every timer that is due, with one save for them (an app with states - * only: a timer can change nothing else). The milliseconds until the next - * is due, or -1 without timers. Missed ticks are not made up. */ -static long run_timers(Executor *ex) { - const PureModule *m = ex->module; - if (!m->states) return -1; - double now = wall_ms(), soonest = -1; - bool ticked = false; - for (uint32_t i = 0; i < m->inputs; ++i) { - if (m->input[i].kind != PURE_KIND_TIMER) continue; - if (!ex->next[i]) ex->next[i] = now + m->input[i].period; /* the first period starts now */ - if (ex->next[i] <= now) { - ex->app.now = now; - if (pure_app_tick(&ex->app, i) == PURE_OK) ++ex->ticks; - ticked = true; - ex->next[i] = now + m->input[i].period; - } - if (soonest < 0 || ex->next[i] - now < soonest) soonest = ex->next[i] - now; - } - if (ticked && pure_app_save(&ex->app) == PURE_OK) after_save(ex); /* a failed save undoes the ticks */ - return soonest < 0 ? -1 : (long)ceil(soonest); -} - -/* On the request's connection thread: complete its ticket and answer it if - * its connection still waits for it. */ -static void answer_applied(Host *host, Applied *a) { - if (!pure_runtime_complete(a->ticket, a->status, a->input == PURE_INPUT_READ ? &a->resource : NULL, - a->status == PURE_OK ? a->reply : (PureSpan){0}, a->revision)) abort(); - Connection *c = host->by_slot[a->ticket.slot]; - if (c && c->phase == WAITING && c->ticket.generation == a->ticket.generation) { - PureOutcome result; - if (!pure_runtime_result(a->ticket, &result)) abort(); - /* A lent answer's body is its extents (the runtime charged them). */ - if (result.status == PURE_OK && a->extents) - respond_typed(c, 200, "OK", "none", result.revision, a->type ? a->type : "application/octet-stream", - a->extents, a->extent_count, result.body.size); - else if (a->route != ROUTE_OBJECT) answer_app(c, result, a->route == ROUTE_VIEW, a->fault, a->type); - else answer(c, result); - drive(c); /* most answers fit the socket buffer at once */ - } - free(a->extents); -} - -static void answer_burst(void *context, Applied *applied, size_t count) { - (void)context; - for (size_t i = 0; i < count; ++i) answer_applied(applied[i].host, &applied[i]); -} - -/* On the owner: give each connection thread its applied requests, and wake it. */ -static void hand_back(void *context, Applied *applied, size_t count) { - Owner *owner = context; - for (unsigned h = 0; h < owner->host_count; ++h) { - Host *host = &owner->hosts[h]; - size_t mine = 0; - for (size_t i = 0; i < count; ++i) mine += applied[i].host == host; - if (!mine) continue; - pthread_mutex_lock(&host->lock); - if (!grow(&host->done, &host->done_capacity, host->done_count + mine, sizeof(*host->done))) abort(); - for (size_t i = 0; i < count; ++i) - if (applied[i].host == host) host->done[host->done_count++] = applied[i]; - pthread_mutex_unlock(&host->lock); - uint64_t one = 1; - if (write(host->wake, &one, sizeof(one)) != sizeof(one)) abort(); - } -} - -/* On a connection thread: answer what the owner handed back. */ -static void answer_handed(Host *host) { - uint64_t signals; - if (read(host->wake, &signals, sizeof(signals)) < 0 && errno != EAGAIN) abort(); - pthread_mutex_lock(&host->lock); - Applied *done = host->done; - size_t count = host->done_count, capacity = host->done_capacity; - host->done = host->answering; - host->done_capacity = host->answering_capacity; - host->done_count = 0; - pthread_mutex_unlock(&host->lock); - host->answering = done; - host->answering_capacity = capacity; - for (size_t i = 0; i < count; ++i) answer_applied(host, &done[i]); - host->pending -= count; -} - -/* The owner's newest snapshot, held, shown by this thread's reader; NULL - * when there is none. Taken for each round, after the round's answers from - * the owner: those were handed back after their saves' snapshots. */ -static PureSnapshot *read_published(Host *host) { - Executor *ex = &host->owner->executor; - pthread_mutex_lock(&ex->published_lock); - PureSnapshot *snapshot = ex->published; - if (snapshot) pure_snapshot_hold(snapshot); - pthread_mutex_unlock(&ex->published_lock); - if (snapshot) pure_app_read(&host->reader, snapshot); - return snapshot; -} - -/* Every request made ready since the last round: run here, or handed to the - * owner. Views are shown here from the snapshot when there is one, after the - * rest went to the owner, which meanwhile applies and saves them. Answering - * may make more ready; they form the next round. */ -static void serve_bursts(Host *host) { - for (;;) { - size_t count = 0; - PureEffect effect; - while (pure_runtime_take(&host->runtime, &effect) == PURE_TAKE_EFFECT) - host->jobs[count++] = (Job){host, effect, host->routes[effect.ticket.slot]}; - if (!count) return; - if (!host->owner) { - execute(&host->local, host->jobs, count, answer_burst, NULL); - continue; - } - PureSnapshot *snapshot = NULL; - size_t views = 0; - for (size_t i = 0; i < count; ++i) views += host->jobs[i].route.kind == ROUTE_VIEW; - if (views && !(snapshot = read_published(host))) views = 0; /* the owner shows them */ - size_t handed = count - views; - if (handed) { - Owner *owner = host->owner; - pthread_mutex_lock(&owner->lock); - if (!grow(&owner->jobs, &owner->capacity, owner->count + handed, sizeof(*owner->jobs))) abort(); - for (size_t i = 0; i < count; ++i) - if (!snapshot || host->jobs[i].route.kind != ROUTE_VIEW) owner->jobs[owner->count++] = host->jobs[i]; - owner->arrivals += handed; - pthread_cond_signal(&owner->waiting); - pthread_mutex_unlock(&owner->lock); - host->pending += handed; - } - if (!views) return; - /* Shown first and answered after, as a burst is. */ - if (!grow(&host->views, &host->views_capacity, views, sizeof(*host->views))) abort(); - size_t shown = 0; - for (size_t i = 0; i < count; ++i) { - const Job *job = &host->jobs[i]; - if (job->route.kind != ROUTE_VIEW) continue; - Applied *a = &host->views[shown++]; - *a = (Applied){.host = host, .ticket = job->effect.ticket, .input = job->effect.input, .route = ROUTE_VIEW}; - a->status = show_view(&host->reader, &job->route, job->effect.available, a); - } - pure_snapshot_release(snapshot); /* what answers lend holds its own mapping */ - for (size_t i = 0; i < shown; ++i) answer_applied(host, &host->views[i]); - host->shown += shown; - } -} - -/* The owner's thread: take every waiting request of every connection thread - * as one round, until they have all stopped. */ -static struct timespec later(struct timespec t, long nanoseconds) { - t.tv_sec += nanoseconds / 1000000000; - t.tv_nsec += nanoseconds % 1000000000; - if (t.tv_nsec >= 1000000000) { ++t.tv_sec; t.tv_nsec -= 1000000000; } - return t; -} - -static bool before(struct timespec a, struct timespec b) { - return a.tv_sec < b.tv_sec || (a.tv_sec == b.tv_sec && a.tv_nsec < b.tv_nsec); -} - -static void *run_owner(void *context) { - Owner *owner = context; - Job *spare = NULL; - size_t spare_capacity = 0; - bool gather = false; - uint64_t answered_at = 0; /* arrivals when the last round's answers were handed back */ - for (;;) { - pthread_mutex_lock(&owner->lock); - /* Wait for requests, or until the next timer is due; trim what the - * wires delivered meanwhile. */ - for (;;) { - bool delivered = owner->delivered; - owner->delivered = false; - pthread_mutex_unlock(&owner->lock); - long due = run_timers(&owner->executor); - bool answered = delivered && take_deliveries(&owner->executor); - pthread_mutex_lock(&owner->lock); - /* Answers are saved with the next round, or now if none waits. */ - if (answered && !owner->count && !owner->stopping) { - pthread_mutex_unlock(&owner->lock); - if (pure_app_save(&owner->executor.app) == PURE_OK) after_save(&owner->executor); - pthread_mutex_lock(&owner->lock); - } - if (owner->count || owner->stopping) break; - if (owner->delivered) continue; - if (due < 0) { - pthread_cond_wait(&owner->waiting, &owner->lock); - continue; - } - struct timespec wake; - clock_gettime(CLOCK_REALTIME, &wake); - wake = later(wake, due * 1000000L); - pthread_cond_timedwait(&owner->waiting, &owner->lock, &wake); - } - /* After a round that saved, its answers bring requests back from the - * connection threads: gather them until none has come for 100 µs (a - * millisecond at most), so they share one save as they would on a - * single thread, where answering takes that long anyway. The quiet - * counts only once a request came after the answers: requests that - * waited out the save alone would otherwise start the next round - * without the clients just answered, and the clients would settle - * into two halves taking turns, each save carrying half of them. */ - struct timespec now, end; - clock_gettime(CLOCK_REALTIME, &now); - end = later(now, 1000000); - while (gather && owner->count && !owner->stopping && before(now, end)) { - size_t seen = owner->count; - bool returned = owner->arrivals != answered_at; - struct timespec quiet = later(now, 100000); - pthread_cond_timedwait(&owner->waiting, &owner->lock, returned && before(quiet, end) ? &quiet : &end); - if (returned && owner->count == seen) break; - clock_gettime(CLOCK_REALTIME, &now); - } - if (!owner->count) { - pthread_mutex_unlock(&owner->lock); - break; - } - Job *jobs = owner->jobs; - size_t count = owner->count, capacity = owner->capacity; - owner->jobs = spare; - owner->capacity = spare_capacity; - owner->count = 0; - pthread_mutex_unlock(&owner->lock); - owner->executor.wrote = false; - execute(&owner->executor, jobs, count, hand_back, owner); - gather = owner->executor.wrote; - pthread_mutex_lock(&owner->lock); - answered_at = owner->arrivals; - pthread_mutex_unlock(&owner->lock); - spare = jobs; - spare_capacity = capacity; - } - free(spare); - return NULL; -} - -/* A write's reply is borrowed from its request, so the handler of an input - * served by PUT or POST may reply only a field of its message. */ -static bool replies_borrow(const PureModule *m, uint32_t input) { - for (uint32_t h = 0; h < m->handlers; ++h) { - if (m->handler[h].event != 1 + 2 * input) continue; - const PureBlock *code = &m->handler[h].code; - /* A field replied as it is, or a text field as its bytes. */ - for (uint32_t pc = code->start; pc < code->start + code->length; ++pc) { - if (m->code[pc].op != PURE_OP_REPLY) continue; - uint32_t from = pc > code->start && m->code[pc - 1].op == PURE_OP_BYTES_FROM ? pc - 1 : pc; - if (from == code->start || m->code[from - 1].op != PURE_OP_FIELD) return false; - } - } - return true; -} - -/* ---- Main ------------------------------------------------------------- */ - -static bool load_module(PureModule *m, const char *path, bool *documents, uint32_t *show_out, uint32_t *edit_out) { - char error[160]; - if (pure_module_read(path, m, error, sizeof(error)) != PURE_OK) { - fprintf(stderr, "Module: %s\n", error); - return false; - } - /* Documents are served when the module has show and edit, as keyed messages. */ - int show = pure_module_input(m, "show"), edit = pure_module_input(m, "edit"); - *documents = show >= 0 || edit >= 0; - if (*documents && (show < 0 || edit < 0 || m->input[show].kind != PURE_KIND_MESSAGE || !m->input[show].keyed || - m->input[edit].kind != PURE_KIND_MESSAGE || !m->input[edit].keyed)) { - fputs("Module: documents need keyed messages named show and edit\n", stderr); - return false; - } - for (uint32_t i = 0; i < m->inputs; ++i) { - bool served = (*documents && i == (uint32_t)edit) || (m->input[i].kind == PURE_KIND_MESSAGE && !m->input[i].keyed); - if (served && !replies_borrow(m, i)) { - fprintf(stderr, "Module: %s may reply only a field of its message\n", m->input_names[i]); - return false; - } - } - *show_out = (uint32_t)(show >= 0 ? show : 0); - *edit_out = (uint32_t)(edit >= 0 ? edit : 0); - return true; -} - -static void accept_all(Host *host) { - for (;;) { - int fd = accept4(host->listener, NULL, NULL, SOCK_NONBLOCK | SOCK_CLOEXEC); - if (fd < 0) return; - if (host->active == host->capacity) { close(fd); continue; } /* full: the peer sees the close */ - Connection *c = host->spare; - if (c) host->spare = c->next; - else if (!(c = malloc(sizeof(*c)))) { close(fd); continue; } - size_t index = 0; - while (host->table[index]) ++index; /* a free index exists: active < capacity */ - int one = 1; - setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); - c->host = host; - c->index = index; - c->fd = fd; - c->phase = READING_HEAD; - c->readable = c->writable = c->keep_alive = true; - c->deadline = now_ms() + host->timeout_ms; - c->ticket = (PureTicket){0}; - c->out = NULL; - c->out_count = c->out_at = 0; - c->body = NULL; - c->in_used = 0; - struct epoll_event event = {.events = EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET, .data.ptr = c}; - if (epoll_ctl(host->epoll, EPOLL_CTL_ADD, fd, &event)) { - close(fd); - c->next = host->spare; - host->spare = c; - continue; - } - host->table[index] = c; - ++host->active; - ++host->accepted; - drive(c); - } -} - -static bool start_executor(Executor *ex, const PureModule *module, uint32_t show, uint32_t edit, const char *store) { - ex->module = module; - ex->show = show; - ex->edit = edit; - if (pure_store_open(store, &ex->store) != PURE_OK) { fputs("Cannot open the store\n", stderr); return false; } - if (pure_app_start(&ex->app, module, pure_store_data(ex->store)) != PURE_OK || !pure_app_defer(&ex->app, true)) { - fputs("Cannot start the module on the store\n", stderr); - return false; - } - return true; -} - -static void stop_executor(Executor *ex) { - pure_snapshot_release(ex->published); - pure_app_stop(&ex->app); - pure_store_close(ex->store); - free(ex->applied); -} - -/* One connection thread: its own listener on the shared port, epoll set, - * runtime domain and, unless an owner runs the app for all of them, app and - * store handle. */ -static bool start_shard(Host *host, const PureModule *module, bool documents, uint32_t show, uint32_t edit, - const char *store, size_t budget, unsigned *port) { - host->module = module; - host->documents = documents; - host->wake = -1; - if (!host->owner && !start_executor(&host->local, module, show, edit, store)) return false; - host->work = calloc(host->capacity, sizeof(*host->work)); - host->table = calloc(host->capacity, sizeof(*host->table)); - host->by_slot = calloc(host->capacity, sizeof(*host->by_slot)); - host->jobs = calloc(host->capacity, sizeof(*host->jobs)); - host->routes = calloc(host->capacity, sizeof(*host->routes)); - if (!host->work || !host->table || !host->by_slot || !host->jobs || !host->routes) { - fputs("Cannot allocate connections\n", stderr); - return false; - } - if (!pure_runtime_init(&host->runtime, host->work, host->capacity, host, (PureHost){.allocate = allocate_ingress}, - budget) || - !pure_runtime_enable_async(&host->runtime)) return false; - host->listener = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0); - int one = 1; - setsockopt(host->listener, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); - setsockopt(host->listener, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)); - struct sockaddr_in address = {.sin_family = AF_INET, .sin_port = htons((uint16_t)*port), - .sin_addr.s_addr = htonl(INADDR_LOOPBACK)}; - socklen_t address_size = sizeof(address); - if (host->listener < 0 || bind(host->listener, (struct sockaddr *)&address, sizeof(address)) || - listen(host->listener, 4096) || getsockname(host->listener, (struct sockaddr *)&address, &address_size)) { - perror("listen"); - return false; - } - *port = ntohs(address.sin_port); /* the first shard picks it, the others join it */ - host->epoll = epoll_create1(EPOLL_CLOEXEC); - struct epoll_event listen_event = {.events = EPOLLIN, .data.ptr = NULL}; - if (host->epoll < 0 || epoll_ctl(host->epoll, EPOLL_CTL_ADD, host->listener, &listen_event)) { - perror("epoll"); - return false; - } - /* The owner says it handed requests back through an eventfd. */ - if (host->owner) { - struct epoll_event wake_event = {.events = EPOLLIN, .data.ptr = &host->wake}; - if (pthread_mutex_init(&host->lock, NULL) || (host->wake = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC)) < 0 || - epoll_ctl(host->epoll, EPOLL_CTL_ADD, host->wake, &wake_event)) { - perror("eventfd"); - return false; - } - } - return true; -} - -static void *run_shard(void *context) { - Host *host = context; - struct epoll_event events[EVENTS]; - uint64_t next_sweep = now_ms() + 1000; - while (!atomic_load(&stopping)) { - /* A thread that runs the app ticks its timers on time. */ - long due = host->owner ? -1 : run_timers(&host->local); - int ready = epoll_wait(host->epoll, events, EVENTS, due >= 0 && due < 1000 ? (int)due : 1000); - for (int e = 0; e < ready; ++e) { - Connection *c = events[e].data.ptr; - if (!c) { accept_all(host); continue; } - if (events[e].data.ptr == &host->wake) { answer_handed(host); continue; } - if (c->phase == CLOSED) continue; - if (events[e].events & EPOLLERR) { close_connection(c); continue; } - if (events[e].events & (EPOLLIN | EPOLLRDHUP | EPOLLHUP)) c->readable = true; - if (events[e].events & EPOLLOUT) c->writable = true; - drive(c); - } - serve_bursts(host); - uint64_t now = now_ms(); - if (now >= next_sweep) { - next_sweep = now + 1000; - for (size_t i = 0; i < host->capacity; ++i) { - Connection *c = host->table[i]; - if (c && c->phase != WAITING && now >= c->deadline) { ++host->expired; close_connection(c); } - } - } - /* The round is over: closed connections can be reused. */ - while (host->closed) { - Connection *c = host->closed; - host->closed = c->next; - c->next = host->spare; - host->spare = c; - } - } - pure_runtime_stop(&host->runtime); - serve_bursts(host); - /* What the owner still runs is answered before the connections close. */ - while (host->pending) { - struct pollfd wake = {host->wake, POLLIN, 0}; - if (poll(&wake, 1, -1) > 0) answer_handed(host); - } - for (size_t i = 0; i < host->capacity; ++i) if (host->table[i]) close_connection(host->table[i]); - return NULL; -} - -static void stop_shard(Host *host) { - if (!host->owner) stop_executor(&host->local); - pure_app_stop(&host->reader); - if (host->listener >= 0) close(host->listener); - if (host->epoll >= 0) close(host->epoll); - if (host->wake >= 0) close(host->wake); - while (host->closed) { - Connection *c = host->closed; - host->closed = c->next; - c->next = host->spare; - host->spare = c; - } - for (Connection *c = host->spare, *next; c; c = next) { next = c->next; free(c); } - free(host->work); - free(host->table); - free(host->by_slot); - free(host->jobs); - free(host->routes); - free(host->done); - free(host->answering); - free(host->views); -} - -/* A wire delivered a send: the owner trims it. */ -static void wire_delivered(void *context) { - Owner *owner = context; - pthread_mutex_lock(&owner->lock); - owner->delivered = true; - pthread_cond_signal(&owner->waiting); - pthread_mutex_unlock(&owner->lock); -} - -/* --send OUTPUT=URL: that output's saved sends go to url, from NAME.OUTPUT. */ -static bool open_wires(Executor *ex, const PureModule *m, const char *const *sends, unsigned count, const char *name) { - for (unsigned i = 0; i < count; ++i) { - const char *equals = strchr(sends[i], '='); - char output[PURE_MODULE_NAME], source[PURE_SOURCE + 2], error[160]; - size_t length = equals ? (size_t)(equals - sends[i]) : 0; - if (!length || length >= sizeof(output)) { - fprintf(stderr, "--send takes OUTPUT=URL, not %s\n", sends[i]); - return false; - } - memcpy(output, sends[i], length); - output[length] = 0; - int written = snprintf(source, sizeof(source), "%s.%s", name, output); - PureWire *wire = written > 0 && written <= PURE_SOURCE ? pure_wire_open(m, output, equals + 1, source, error, sizeof(error)) : NULL; - if (!wire) { - fprintf(stderr, "--send %s: %s\n", sends[i], written > PURE_SOURCE ? "the name and output are longer than 64 bytes" : error); - return false; - } - for (unsigned k = 0; k < ex->wire_count; ++k) - if (pure_wire_output(ex->wires[k]) == pure_wire_output(wire)) { - fprintf(stderr, "--send %s: the output is wired already\n", sends[i]); - pure_wire_close(wire); - return false; - } - ex->wires[ex->wire_count++] = wire; - } - return true; -} - -/* --reply OUTPUT=INPUT: the answers to that output's sends become INPUT's events. */ -static bool wire_replies(Executor *ex, const PureModule *m, const char *const *replies, unsigned count) { - for (unsigned i = 0; i < count; ++i) { - const char *equals = strchr(replies[i], '='); - char output[PURE_MODULE_NAME], error[160] = ""; - size_t length = equals ? (size_t)(equals - replies[i]) : 0; - int input = equals ? pure_module_input(m, equals + 1) : -1; - PureWire *wire = NULL; - if (length && length < sizeof(output)) { - memcpy(output, replies[i], length); - output[length] = 0; - int index = pure_module_input(m, output); - for (unsigned k = 0; k < ex->wire_count && index >= 0; ++k) - if (pure_wire_output(ex->wires[k]) == (uint32_t)index) wire = ex->wires[k]; - } - if (!wire || input < 0 || !pure_wire_reply(wire, (uint32_t)input, error, sizeof(error))) { - fprintf(stderr, "--reply %s: %s\n", replies[i], !wire ? "takes OUTPUT=INPUT of an output given --send" : - input < 0 ? "no such input" : error); - return false; - } - } - return true; -} - -/* --threads auto: a connection thread for each CPU this process may run on, - * less the one the owner of an app with states takes. */ -static unsigned auto_threads(bool owner) { - cpu_set_t set; - long cpus = sched_getaffinity(0, sizeof(set), &set) ? sysconf(_SC_NPROCESSORS_ONLN) : CPU_COUNT(&set); - if (cpus < 1) cpus = 1; - if (owner && cpus > 1) --cpus; - return cpus > 256 ? 256 : (unsigned)cpus; -} - -int main(int argc, char **argv) { - static PureModule module; - static Owner owner; - const char *module_path = NULL, *store = NULL, *name = NULL, *sends[PURE_MODULE_INPUTS], *replies[PURE_MODULE_INPUTS]; - unsigned send_count = 0, reply_count = 0; - unsigned port = 8767, threads = 1; - size_t budget = 64u << 20, capacity = 1024; - uint64_t timeout_ms = 10000; - for (int i = 1; i < argc; i += 2) { - uint64_t value; - if (i + 1 == argc) goto usage; - if (!strcmp(argv[i], "--module")) module_path = argv[i + 1]; - else if (!strcmp(argv[i], "--store")) store = argv[i + 1]; - else if (!strcmp(argv[i], "--send") && send_count < PURE_MODULE_INPUTS) sends[send_count++] = argv[i + 1]; - else if (!strcmp(argv[i], "--name")) name = argv[i + 1]; - else if (!strcmp(argv[i], "--send-ca")) pure_wire_trust(argv[i + 1]); - else if (!strcmp(argv[i], "--reply") && reply_count < PURE_MODULE_INPUTS) replies[reply_count++] = argv[i + 1]; - else if (!strcmp(argv[i], "--port") && number(argv[i + 1], 65535, &value)) port = (unsigned)value; - else if (!strcmp(argv[i], "--budget") && number(argv[i + 1], SIZE_MAX, &value) && value) budget = (size_t)value; - else if (!strcmp(argv[i], "--connections") && number(argv[i + 1], 1u << 20, &value) && value) capacity = (size_t)value; - else if (!strcmp(argv[i], "--threads") && !strcmp(argv[i + 1], "auto")) threads = 0; - else if (!strcmp(argv[i], "--threads") && number(argv[i + 1], 256, &value) && value) threads = (unsigned)value; - else if (!strcmp(argv[i], "--timeout-ms") && number(argv[i + 1], 3600000, &value) && value) timeout_ms = value; - else goto usage; - } - uint32_t show, edit; - bool documents; - if (!module_path || !store || !load_module(&module, module_path, &documents, &show, &edit)) goto usage; - if (!threads) threads = auto_threads(module.states); - if (pure_store_init(store) != PURE_OK) { fputs("Cannot initialize the store directory\n", stderr); return 1; } - signal(SIGPIPE, SIG_IGN); - struct sigaction stop_action = {.sa_handler = stop}; - sigemptyset(&stop_action.sa_mask); - sigaction(SIGINT, &stop_action, NULL); - sigaction(SIGTERM, &stop_action, NULL); - Host *hosts = calloc(threads, sizeof(*hosts)); - pthread_t *thread = calloc(threads, sizeof(*thread)), owner_thread; - if (!hosts || !thread) return 1; - /* States (and outputs, whose sends are the app's state) have one owner on - * a thread of its own, whose snapshots the connection threads read. */ - bool outputs = false; - for (uint32_t i = 0; i < module.inputs; ++i) outputs = outputs || module.input[i].kind == PURE_KIND_OUTPUT; - bool owned = module.states || outputs; - /* The sender's name: --name, or the module file's name without .pmod. */ - char stem[PURE_SOURCE + 1]; - if (!name) { - const char *base = strrchr(module_path, '/') ? strrchr(module_path, '/') + 1 : module_path; - size_t length = strlen(base); - if (length > 5 && !strcmp(base + length - 5, ".pmod")) length -= 5; - snprintf(stem, sizeof(stem), "%.*s", (int)length, base); - name = stem; - } - if (owned) { - if (!start_executor(&owner.executor, &module, show, edit, store) || pthread_mutex_init(&owner.lock, NULL) || - pthread_cond_init(&owner.waiting, NULL) || pthread_mutex_init(&owner.executor.published_lock, NULL)) - return 1; - owner.executor.reads_first = true; - owner.executor.publishes = true; - if (!open_wires(&owner.executor, &module, sends, send_count, name) || - !wire_replies(&owner.executor, &module, replies, reply_count)) return 2; - after_save(&owner.executor); /* the snapshot, and the sends stored before */ - for (unsigned i = 0; i < owner.executor.wire_count; ++i) - if (!pure_wire_start(owner.executor.wires[i], wire_delivered, &owner)) { fputs("Cannot start a wire\n", stderr); return 1; } - owner.hosts = hosts; - owner.host_count = threads; - } - for (unsigned i = 0; i < threads; ++i) { - hosts[i].capacity = capacity; - hosts[i].timeout_ms = timeout_ms; - hosts[i].listener = hosts[i].epoll = -1; - hosts[i].owner = owned ? &owner : NULL; - if (!start_shard(&hosts[i], &module, documents, show, edit, store, budget, &port)) return 1; - } - if (owned && pthread_create(&owner_thread, NULL, run_owner, &owner)) { fputs("Cannot start the owner\n", stderr); return 1; } - printf("{\"port\":%u,\"threads\":%u}\n", port, threads); - fflush(stdout); - for (unsigned i = 1; i < threads; ++i) - if (pthread_create(&thread[i], NULL, run_shard, &hosts[i])) { fputs("Cannot start a thread\n", stderr); return 1; } - run_shard(&hosts[0]); - for (unsigned i = 1; i < threads; ++i) pthread_join(thread[i], NULL); - if (owned) { - pthread_mutex_lock(&owner.lock); - owner.stopping = true; - pthread_cond_signal(&owner.waiting); - pthread_mutex_unlock(&owner.lock); - pthread_join(owner_thread, NULL); - } - size_t active = 0, retained = 0; - uint64_t accepted = 0, requests = 0, bursts = owner.executor.bursts, events = owner.executor.burst_events, expired = 0, - ticks = owner.executor.ticks, shown = 0; - for (unsigned i = 0; i < threads; ++i) { - shown += hosts[i].shown; - active += hosts[i].active; - retained += hosts[i].runtime.retained; - accepted += hosts[i].accepted; - requests += hosts[i].requests; - bursts += hosts[i].local.bursts; - events += hosts[i].local.burst_events; - ticks += hosts[i].local.ticks; - expired += hosts[i].expired; - stop_shard(&hosts[i]); - } - uint64_t delivered = 0, attempts = 0; - for (unsigned i = 0; i < owner.executor.wire_count; ++i) { - uint64_t d, a; - pure_wire_counts(owner.executor.wires[i], &d, &a); - delivered += d; - attempts += a; - pure_wire_close(owner.executor.wires[i]); /* what was not delivered stays in the outbox */ - } - if (owned) { - stop_executor(&owner.executor); - free(owner.jobs); - } - unsigned long groups, grouped; - pure_store_group_counts(&groups, &grouped); - fprintf(stderr, "{\"active\":%zu,\"retained_payload_bytes\":%zu,\"accepted\":%" PRIu64 ",\"requests\":%" PRIu64 - ",\"bursts\":%" PRIu64 ",\"burst_events\":%" PRIu64 ",\"ticks\":%" PRIu64 ",\"expired\":%" PRIu64 - ",\"threads\":%u,\"owner\":%s,\"snapshots\":%" PRIu64 ",\"views_from_snapshots\":%" PRIu64 - ",\"delivered\":%" PRIu64 ",\"delivery_attempts\":%" PRIu64 ",\"answers\":%" PRIu64 - ",\"commit_groups\":%lu,\"grouped_commits\":%lu}\n", - active, retained, accepted, requests, bursts, events, ticks, expired, threads, owned ? "true" : "false", - owner.executor.publications, shown, delivered, attempts, owner.executor.answers, groups, grouped); - free(hosts); - free(thread); - return 0; -usage: - fputs("pure-serve --module FILE.pmod --store DIRECTORY [--port 8767] [--threads 1|N|auto] [--budget BYTES] " - "[--connections 1024] [--timeout-ms 10000] [--send OUTPUT=http[s]://host:port/path ...] [--name NAME] " - "[--send-ca FILE] [--reply OUTPUT=INPUT ...]\n", stderr); - return 2; -} diff --git a/prototypes/connections/state.h b/prototypes/connections/state.h deleted file mode 100644 index 2bf78b0..0000000 --- a/prototypes/connections/state.h +++ /dev/null @@ -1,84 +0,0 @@ -#ifndef PURE_STATE_H -#define PURE_STATE_H -#include "flow.h" -#include "result.h" -#include - -/* Runtime-owned working versions of stored values (selected tree store). - * A draft starts at the committed version of a key and changes only in - * memory: an edit copies the leaf and path it touches and shares every other - * extent with the stored version. An app therefore keeps no copy of the value - * and never compares a copy with storage to find what it changed. Commit - * appends only the draft's new extents. One owner thread per draft. */ -typedef struct PureDraft PureDraft; - -/* A working value with no stored base, for runtime-owned state such as an - * app's text. It edits and reads like any draft but cannot be committed. */ -PureDraft *pure_draft_new(void); -/* A missing key opens as an empty draft at revision 0; commit creates it. */ -PureResult pure_draft_open(const char *path, const char *key, PureDraft **out); -/* A runtime-owned value of an app, such as a list or text state, stored - * under a name that is never a document key (hosts listing documents do not - * see it). name is a valid key of at most 63 bytes. */ -PureResult pure_state_open(const char *path, const char *name, PureDraft **out); -/* A store kept open by one owner thread. Drafts opened through it share one - * mapping of the newest version, and opening one checks only whether the file - * changed; commits of its drafts map the new version once. It must outlive - * its drafts. */ -typedef struct PureStore PureStore; -PureResult pure_store_open(const char *path, PureStore **out); -void pure_store_close(PureStore *); -PureResult pure_store_draft(PureStore *, const char *key, PureDraft **out); -PureResult pure_store_state(PureStore *, const char *name, PureDraft **out); -/* Stored extents can outlive their draft: pure_draft_mapping takes a - * reference to the mapping a draft opened through a handle reads its stored - * nodes from (NULL for other drafts). Spans that mapping holds stay readable - * until the reference is released, after the draft closes and after the - * store moves on. Spans of a draft's unsaved edits are not in it. Take it on - * the handle's owner thread (a frozen draft's on any); the reference may be - * released on any thread. */ -typedef struct PureMapping PureMapping; -PureMapping *pure_draft_mapping(const PureDraft *); -bool pure_mapping_holds(const PureMapping *, PureSpan); -void pure_mapping_release(PureMapping *); -/* Pinned, drafts open at the version found when pinning, without checking - * the file each time (a host pins for one burst of requests); commits made - * through the handle still move it on. */ -PureResult pure_store_pin(PureStore *, bool pinned); -/* Memory only. Fails without change on a bad range or when memory runs out. */ -PureResult pure_draft_splice(PureDraft *, size_t offset, size_t remove, PureSpan insert); -size_t pure_draft_size(const PureDraft *); -/* The committed revision this draft is based on; 0 if the key was absent. */ -int64_t pure_draft_revision(const PureDraft *); -/* Visit [offset, offset + length) in order as borrowed extents, without - * copying. Returning false stops early. Spans stay valid until the next - * splice, commit or close of this draft. */ -PureResult pure_draft_read(const PureDraft *, size_t offset, size_t length, - bool (*visit)(void *, PureSpan), void *context); -/* An immutable version of a draft's value. It stays readable while the draft - * is held (reclamation is suspended) and until the next commit or close. */ -typedef struct { uint64_t ref, bytes; } PureVersion; -PureVersion pure_draft_version(const PureDraft *); -PureResult pure_draft_read_version(const PureDraft *, PureVersion, size_t offset, size_t length, - bool (*visit)(void *, PureSpan), void *context); -/* A read-only copy of a draft opened through a store handle, at its current - * version, when that version is all stored (no edits since its last commit, - * as right after one): it shares the draft's mapping, and since reading - * changes nothing it may be read, lend its mapping and close on any thread. - * NULL otherwise, or when memory runs out. */ -PureDraft *pure_draft_freeze(const PureDraft *); -/* While held, edits keep every earlier version's nodes (one transaction). */ -void pure_draft_hold(PureDraft *, bool held); -/* Abandon edits made since version was taken; the draft was held since then. */ -void pure_draft_restore(PureDraft *, PureVersion); -/* Publish if the key is still at the draft's base revision. On success the - * draft continues from the new revision; on any failure it is unchanged. */ -PureResult pure_draft_commit(PureDraft *, int64_t *revision); -/* Publish drafts of one store in one commit: each key must still be at its - * draft's base revision (a conflict names the stored revision in revisions), - * and either every draft is published or none is. Keys must differ. */ -PureResult pure_draft_commit_all(PureDraft *const *drafts, size_t count, int64_t *revisions); -/* Commits made so far in this process: groups formed, and requests in them. */ -void pure_store_group_counts(unsigned long *groups, unsigned long *requests); -void pure_draft_close(PureDraft *); -#endif diff --git a/prototypes/connections/store-tree.c b/prototypes/connections/store-tree.c deleted file mode 100644 index 146e919..0000000 --- a/prototypes/connections/store-tree.c +++ /dev/null @@ -1,1856 +0,0 @@ -/* Selected storage experiment: every document lives in one append-only file as - * a persistent B-tree of byte extents. An edit writes only the leaves and the - * path it changes, so inserting or deleting in a large document costs O(log n) - * nodes instead of a whole-document copy. Unchanged extents are shared by old - * and new versions. The document catalog is the same structure holding fixed - * records sorted by key. A commit appends a record (its head: where its nodes - * are and their digest) and then its nodes, and syncs once: one contiguous - * range, which a flash device writes about twice as fast as two. Every few - * commits the head also goes into the older of two slots at the start, where - * a reader begins before following the records after it; readers take no - * lock and map one immutable prefix. */ -#define _POSIX_C_SOURCE 200809L -#ifdef __APPLE__ -#define _DARWIN_C_SOURCE -#endif -#include "store.h" -#include "state.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { FREE = 0xffff }; -/* Each head slot has a block of its own: a torn write of one never touches - * the other. */ -enum { SLOT = 128, BLOCK = 4096, DATA = 2 * BLOCK, LEAF_MAX = 4096, LEAF_MIN = 1024, - FAN_MAX = 32, FAN_MIN = 8, RECORD = 96, KEY = 64, HEIGHT_MAX = 12, - BUFFER = 65536 }; -/* Written zeros kept after the published end (see prepare_tail). */ -enum { TAIL_MIN = 65536, TAIL_STEP = 262144 }; -#ifdef PURE_STORE_TEST -enum { COMPACT_SLACK = 65536 }; -#else -enum { COMPACT_SLACK = 1048576 }; -#endif -#define LEAF_BIT (UINT64_C(1) << 63) -#define MEMORY_BIT (UINT64_C(1) << 62) -#define LOCATION (MEMORY_BIT - 1) -/* Slots of this format, and of the one before tail records (still read). */ -static const unsigned char head_magic[8] = {'P','U','R','E','T','R','E','3'}; -static const unsigned char earlier_magic[8] = {'P','U','R','E','T','R','E','2'}; -static const unsigned char record_magic[8] = {'P','U','R','E','R','E','C','1'}; -/* Commits between two heads written to a slot. */ -enum { CHECKPOINT = 16 }; -static const unsigned char branch_magic[4] = {'P','T','B','1'}; -static const char tree_name[] = "pure.tree"; -/* Threads of one process take turns as the writer: the file lock arbitrates - * only between processes (it is the process's, and closing any descriptor of - * the lock file drops it). */ -static pthread_mutex_t writers = PTHREAD_MUTEX_INITIALIZER; -static uint64_t temporary_sequence; - -/* A reference is a file offset, or an index into this edit's node arena. - * bytes is the size of the whole subtree. Leaves hold raw document bytes. */ -typedef struct { uint64_t ref, bytes; } Entry; -typedef struct { Entry *items; size_t count, capacity; } List; -typedef struct { - unsigned char *bytes; /* leaf only */ - Entry *children; /* branch only */ - size_t size; /* leaf bytes, child count, or next free slot + 1 */ - unsigned height; /* FREE for a reclaimed slot */ -} Node; -/* One mapped version of the file, shared by the drafts of a store handle - * and by whoever still sends extents from it. */ -typedef struct PureMapping { const unsigned char *map; uint64_t end; unsigned refs; } Mapping; -typedef struct { - const unsigned char *map; - uint64_t end; - Mapping *shared; /* map belongs to this shared mapping, not to the tree */ - Node *nodes; - size_t count, capacity, free_head, free_count; /* free_head is slot + 1 */ - size_t bytes; /* payload held by arena nodes, including unreachable ones */ -} Tree; -/* A commit's bytes are start..end (none for a file synced whole before it was - * published); check is their digest. */ -typedef struct { uint64_t seq, end, catalog, catalog_bytes, live, start, check; } Head; -/* The xxHash64 construction without a seed: four multiply-rotate lanes over - * 32-byte stripes, several GB/s. An all-zero Digest is an empty one. */ -typedef struct { uint64_t lane[4], length; unsigned char held[32]; size_t count; } Digest; -typedef struct { - int fd; - uint64_t at; /* file offset of the next byte */ - size_t used; - bool failed; - Digest digest; /* of every byte written */ - unsigned char buffer[BUFFER]; -} Writer; -typedef struct { PureSpan part[3]; size_t index, offset; } Source; - -static bool fault(PureStoreStage stage) { -#ifdef PURE_STORE_TEST - if (pure_store_test_fault(stage)) { errno = EIO; return true; } -#else - (void)stage; -#endif - return false; -} - -static bool valid_key(const char *key) { - if (!key) return false; - size_t length = strlen(key); - if (!length || length > KEY) return false; - for (size_t i = 0; i < length; ++i) { - char b = key[i]; - if (!((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || - (b >= '0' && b <= '9') || b == '-' || b == '_')) return false; - } - return true; -} - -/* Runtime-owned values (app states) live under '.' followed by a valid key. - * Document keys never start with '.', so the two never meet. */ -static bool runtime_key(const char *key) { - return key && key[0] == '.' && strlen(key) <= KEY && valid_key(key + 1); -} - -static uint64_t decode64(const unsigned char *bytes) { - uint64_t value = 0; - for (unsigned i = 0; i < 8; ++i) value |= (uint64_t)bytes[i] << (8 * i); - return value; -} - -static void encode64(unsigned char *bytes, uint64_t value) { - for (unsigned i = 0; i < 8; ++i) bytes[i] = (unsigned char)(value >> (8 * i)); -} - -static uint64_t load64(const unsigned char *bytes) { - uint64_t value; - memcpy(&value, bytes, 8); -#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - value = __builtin_bswap64(value); -#endif - return value; -} - -/* Digests find torn and missing writes; they are not a defence against a - * hostile file. */ -static const uint64_t P1 = UINT64_C(0x9E3779B185EBCA87), P2 = UINT64_C(0xC2B2AE3D27D4EB4F), - P3 = UINT64_C(0x165667B19E3779F9), P4 = UINT64_C(0x85EBCA77C2B2AE63), - P5 = UINT64_C(0x27D4EB2F165667C5); - -static uint64_t rotate(uint64_t value, unsigned bits) { return value << bits | value >> (64 - bits); } -static uint64_t mix(uint64_t lane, uint64_t word) { return rotate(lane + word * P2, 31) * P1; } - -static void stripes(Digest *digest, const unsigned char *bytes, size_t count) { - uint64_t a = digest->lane[0], b = digest->lane[1], c = digest->lane[2], d = digest->lane[3]; - for (; count; --count, bytes += 32) { - a = mix(a, load64(bytes)); - b = mix(b, load64(bytes + 8)); - c = mix(c, load64(bytes + 16)); - d = mix(d, load64(bytes + 24)); - } - digest->lane[0] = a, digest->lane[1] = b, digest->lane[2] = c, digest->lane[3] = d; -} - -static void digest_add(Digest *digest, const unsigned char *bytes, size_t length) { - digest->length += length; - if (digest->count) { - size_t take = 32 - digest->count < length ? 32 - digest->count : length; - memcpy(digest->held + digest->count, bytes, take); - digest->count += take; - bytes += take; - length -= take; - if (digest->count < 32) return; - stripes(digest, digest->held, 1); - digest->count = 0; - } - stripes(digest, bytes, length / 32); - digest->count = length % 32; - if (digest->count) memcpy(digest->held, bytes + length - digest->count, digest->count); -} - -static uint64_t digest_end(const Digest *digest) { - const uint64_t *lane = digest->lane; - uint64_t hash = rotate(lane[0], 1) + rotate(lane[1], 7) + rotate(lane[2], 12) + rotate(lane[3], 18); - for (unsigned i = 0; i < 4; ++i) hash = (hash ^ mix(0, lane[i])) * P1 + P4; - hash += digest->length; - size_t i = 0; - for (; i + 8 <= digest->count; i += 8) hash = rotate(hash ^ mix(0, load64(digest->held + i)), 27) * P1 + P4; - for (; i < digest->count; ++i) hash = rotate(hash ^ digest->held[i] * P5, 11) * P1; - hash ^= hash >> 33; - hash *= P2; - hash ^= hash >> 29; - hash *= P3; - return hash ^ hash >> 32; -} - -static uint64_t digest_of(const unsigned char *bytes, size_t length) { - Digest digest = {0}; - digest_add(&digest, bytes, length); - return digest_end(&digest); -} - -#ifdef PURE_STORE_TEST -static unsigned long syncs; -unsigned long pure_tree_test_syncs(void) { return __atomic_load_n(&syncs, __ATOMIC_RELAXED); } -#define COUNT_SYNC() __atomic_add_fetch(&syncs, 1, __ATOMIC_RELAXED) -#else -#define COUNT_SYNC() ((void)0) -#endif - -static bool sync_fd(int fd) { - int rc; - COUNT_SYNC(); - do { rc = fsync(fd); } while (rc && errno == EINTR); - return rc == 0; -} - -/* Data and the size needed to read it back, without timestamps. */ -static bool sync_data(int fd) { -#ifdef __APPLE__ - return sync_fd(fd); -#else - int rc; - COUNT_SYNC(); - do { rc = fdatasync(fd); } while (rc && errno == EINTR); - return rc == 0; -#endif -} - -static bool write_at(int fd, const unsigned char *bytes, size_t length, uint64_t offset) { - while (length) { - ssize_t n = pwrite(fd, bytes, length, (off_t)offset); - if (n < 0 && errno == EINTR) continue; - if (n <= 0) return false; - bytes += (size_t)n; - length -= (size_t)n; - offset += (uint64_t)n; - } - return true; -} - -static bool read_at(int fd, unsigned char *bytes, size_t length, uint64_t offset) { - while (length) { - ssize_t n = pread(fd, bytes, length, (off_t)offset); - if (n < 0 && errno == EINTR) continue; - if (n <= 0) return false; - bytes += (size_t)n; - length -= (size_t)n; - offset += (uint64_t)n; - } - return true; -} - -static void encode_head(unsigned char out[SLOT], const unsigned char magic[8], const Head *head) { - memset(out, 0, SLOT); - memcpy(out, magic, 8); - encode64(out + 8, head->seq); - encode64(out + 16, head->end); - encode64(out + 24, head->catalog); - encode64(out + 32, head->catalog_bytes); - encode64(out + 40, head->live); - encode64(out + 48, head->start); - encode64(out + 56, head->check); - encode64(out + SLOT - 8, digest_of(out, SLOT - 8)); -} - -/* A slot's head (record false) or a commit record's. */ -static bool decode_head(const unsigned char in[SLOT], bool record, Head *head) { - bool magic = record ? !memcmp(in, record_magic, 8) : !memcmp(in, head_magic, 8) || !memcmp(in, earlier_magic, 8); - if (!magic || decode64(in + SLOT - 8) != digest_of(in, SLOT - 8)) return false; - *head = (Head){decode64(in + 8), decode64(in + 16), decode64(in + 24), decode64(in + 32), - decode64(in + 40), decode64(in + 48), decode64(in + 56)}; - return head->seq && head->end <= (uint64_t)INT64_MAX && head->end <= SIZE_MAX && - head->start >= DATA && head->start <= head->end && head->catalog_bytes % RECORD == 0 && - !head->catalog == !head->catalog_bytes; -} - -/* ---- Trusted heads -------------------------------------------------------- - * One sync makes a commit's bytes and its head durable together, so a crash - * during it can leave the head on the device without all of the bytes: a - * head counts only once its digest matches them. The older slot needs no - * such luck. A writer builds only on a durable head, so the newer head was - * written after the older one's commit was complete on the device. - * The process remembers the heads it published or checked, so it reads a - * commit's bytes at most once, and which of them it made durable. */ -enum { UNKNOWN, BROKEN, CHECKED, DURABLE, PUBLISHING }; -enum { KNOWN = 32 }; -typedef struct { dev_t device; ino_t inode; uint64_t seq, check; int state; } Known; -/* The newest head that counts per file, to follow records from. */ -typedef struct { dev_t device; ino_t inode; Head head; } Newest; -enum { NEWEST = 4 }; -static struct { - pthread_mutex_t lock; - Known heads[KNOWN]; - unsigned next; - Known publishing; /* written here, not durable yet */ - Newest newest[NEWEST]; - unsigned next_newest; -} known = {.lock = PTHREAD_MUTEX_INITIALIZER}; - -static bool names(const Known *known_head, const struct stat *file, const Head *head) { - return known_head->state && known_head->device == file->st_dev && known_head->inode == file->st_ino && - known_head->seq == head->seq && known_head->check == head->check; -} - -static int recall(const struct stat *file, const Head *head) { - int state = UNKNOWN; - pthread_mutex_lock(&known.lock); - if (names(&known.publishing, file, head)) state = PUBLISHING; - for (unsigned i = 0; i < KNOWN && state == UNKNOWN; ++i) - if (names(&known.heads[i], file, head)) state = known.heads[i].state; - pthread_mutex_unlock(&known.lock); - return state; -} - -/* PUBLISHING marks the one head being made durable; any other state ends - * that mark (UNKNOWN only ends it). */ -static void remember(const struct stat *file, const Head *head, int state) { - Known entry = {file->st_dev, file->st_ino, head->seq, head->check, state}; - pthread_mutex_lock(&known.lock); - if (state == PUBLISHING) known.publishing = entry; - else if (names(&known.publishing, file, head)) known.publishing.state = UNKNOWN; - if (state != PUBLISHING && state != UNKNOWN) { - Known *at = NULL; - for (unsigned i = 0; i < KNOWN && !at; ++i) - if (names(&known.heads[i], file, head)) at = &known.heads[i]; - if (!at) known.heads[known.next++ % KNOWN] = entry; - else if (at->state != DURABLE) *at = entry; /* a durable head stays durable */ - } - if (state == CHECKED || state == DURABLE) { - Newest *at = NULL; - for (unsigned i = 0; i < NEWEST && !at; ++i) - if (known.newest[i].head.seq && known.newest[i].device == file->st_dev && - known.newest[i].inode == file->st_ino) at = &known.newest[i]; - if (!at) at = &known.newest[known.next_newest++ % NEWEST]; - else if (at->head.seq >= head->seq) at = NULL; - if (at) *at = (Newest){file->st_dev, file->st_ino, *head}; - } - pthread_mutex_unlock(&known.lock); -} - -static bool newest_known(const struct stat *file, Head *out) { - bool found = false; - pthread_mutex_lock(&known.lock); - for (unsigned i = 0; i < NEWEST && !found; ++i) - if (known.newest[i].head.seq && known.newest[i].device == file->st_dev && known.newest[i].inode == file->st_ino) { - *out = known.newest[i].head; - found = true; - } - pthread_mutex_unlock(&known.lock); - return found; -} - -#ifdef PURE_STORE_TEST -/* Test-only: forget every head, as a new process after a crash would. */ -void pure_tree_test_forget(void) { - pthread_mutex_lock(&known.lock); - memset(known.heads, 0, sizeof(known.heads)); - memset(known.newest, 0, sizeof(known.newest)); - known.publishing.state = UNKNOWN; - pthread_mutex_unlock(&known.lock); -} -#endif - -/* Whether a head's commit has all of its bytes. */ -static bool whole(int fd, const Head *head) { - unsigned char buffer[16384]; - Digest digest = {0}; - for (uint64_t at = head->start; at < head->end;) { - size_t n = head->end - at < sizeof(buffer) ? (size_t)(head->end - at) : sizeof(buffer); - if (!read_at(fd, buffer, n, at)) return false; - digest_add(&digest, buffer, n); - at += n; - } - return digest_end(&digest) == head->check; -} - -/* The newest head that counts. A reader passes over the head this process is - * still making durable, so the process never answers with a commit that a - * crash could take back; a writer never meets one (it is the only writer). */ -static bool pick(int fd, const struct stat *file, const unsigned char *slots[2], bool writer, Head *out, - uint64_t sequences[2]) { - Head heads[2]; - bool valid[2]; - for (unsigned i = 0; i < 2; ++i) { - valid[i] = decode_head(slots[i], false, &heads[i]) && heads[i].end <= (uint64_t)file->st_size; - if (sequences) sequences[i] = valid[i] ? heads[i].seq : 0; - } - unsigned newest = valid[1] && (!valid[0] || heads[1].seq > heads[0].seq); - for (unsigned turn = 0; turn < 2; ++turn) { - unsigned slot = turn ? !newest : newest; - const Head *head = &heads[slot]; - if (!valid[slot]) continue; - int state = recall(file, head); - if (state == BROKEN || (state == PUBLISHING && !writer)) continue; - if (state == UNKNOWN) { - bool counts = whole(fd, head); - remember(file, head, counts ? CHECKED : BROKEN); - if (!counts) continue; - } - *out = *head; - return true; - } - return false; -} - -/* The commit whose record follows head's bytes, if its own bytes are whole. */ -static bool follow(int fd, const struct stat *file, bool writer, const Head *head, Head *out) { - unsigned char record[SLOT]; - uint64_t size = (uint64_t)file->st_size; - Head next; - if (head->end > size || size - head->end < SLOT || !read_at(fd, record, SLOT, head->end) || - !decode_head(record, true, &next) || next.seq != head->seq + 1 || next.start != head->end + SLOT || - next.end > size) return false; - int state = recall(file, &next); - if (state == BROKEN || (state == PUBLISHING && !writer)) return false; - if (state == UNKNOWN) { - bool counts = whole(fd, &next); - remember(file, &next, counts ? CHECKED : BROKEN); - if (!counts) return false; - } - *out = next; - return true; -} - -/* Whether head's record is in the file as it was made. */ -static bool recorded(int fd, const struct stat *file, const Head *head) { - unsigned char stored[SLOT], made[SLOT]; - if (head->start < DATA + SLOT || head->end > (uint64_t)file->st_size) return false; - encode_head(made, record_magic, head); - return read_at(fd, stored, SLOT, head->start - SLOT) && !memcmp(stored, made, SLOT); -} - -/* The newest commit: from the newer slot's head that counts, or from the - * newest this process has seen in the file when its record is still there, - * along the records that follow. sequences gets the slots' (0: none). */ -static bool read_head(int fd, struct stat *file, bool writer, Head *out, uint64_t sequences[2]) { - unsigned char slots[BLOCK + SLOT]; - Head head, seen; - if (fstat(fd, file) || !S_ISREG(file->st_mode) || file->st_size < 0 || !read_at(fd, slots, sizeof(slots), 0) || - !pick(fd, file, (const unsigned char *[2]){slots, slots + BLOCK}, writer, &head, sequences)) return false; - if (newest_known(file, &seen) && seen.seq > head.seq && recorded(fd, file, &seen)) head = seen; - while (follow(fd, file, writer, &head, &head)) {} - *out = head; - return true; -} - -static bool map_tree(int fd, const struct stat *file, const Head *head, Tree *tree) { - *tree = (Tree){0}; - if ((uint64_t)file->st_size < head->end) return false; - void *map = mmap(NULL, (size_t)head->end, PROT_READ, MAP_SHARED, fd, 0); - if (map == MAP_FAILED) return false; - *tree = (Tree){.map = map, .end = head->end}; - return true; -} - -/* A lent mapping may be released on another thread (a host that answers on - * its connection threads): its count is atomic. */ -static void release_mapping(Mapping *mapping) { - if (!mapping || __atomic_sub_fetch(&mapping->refs, 1, __ATOMIC_ACQ_REL)) return; - munmap((void *)mapping->map, (size_t)mapping->end); - free(mapping); -} - -static void release_tree(Tree *tree) { - if (tree->shared) release_mapping(tree->shared); - else if (tree->map) munmap((void *)tree->map, (size_t)tree->end); - for (size_t i = 0; i < tree->count; ++i) { - free(tree->nodes[i].bytes); - free(tree->nodes[i].children); - } - free(tree->nodes); - *tree = (Tree){0}; -} - -static const unsigned char *leaf_bytes(const Tree *tree, Entry entry) { - if (!(entry.ref & LEAF_BIT) || !entry.bytes || entry.bytes > LEAF_MAX) return NULL; - uint64_t at = entry.ref & LOCATION; - if (entry.ref & MEMORY_BIT) { - if (at >= tree->count) return NULL; - const Node *node = &tree->nodes[at]; - return node->bytes && node->size == entry.bytes ? node->bytes : NULL; - } - if (at < DATA || at > tree->end || entry.bytes > tree->end - at) return NULL; - return tree->map + at; -} - -/* Returns the child count of a checked branch at height, or 0. */ -static size_t children(const Tree *tree, Entry entry, unsigned height, Entry out[FAN_MAX]) { - if ((entry.ref & LEAF_BIT) || !height) return 0; - uint64_t at = entry.ref & LOCATION; - size_t count; - if (entry.ref & MEMORY_BIT) { - if (at >= tree->count) return 0; - const Node *node = &tree->nodes[at]; - if (node->bytes || node->height != height || !node->size || node->size > FAN_MAX) return 0; - count = node->size; - memcpy(out, node->children, count * sizeof(*out)); - } else { - if (at < DATA || at % 8 || at > tree->end || tree->end - at < 8) return 0; - const unsigned char *p = tree->map + at; - count = (size_t)p[4] | (size_t)p[5] << 8; - unsigned stored = (unsigned)p[6] | (unsigned)p[7] << 8; - if (memcmp(p, branch_magic, sizeof(branch_magic)) || stored != height || !count || - count > FAN_MAX || (tree->end - at - 8) / 16 < count) return 0; - for (size_t i = 0; i < count; ++i) - out[i] = (Entry){decode64(p + 8 + 16 * i), decode64(p + 16 + 16 * i)}; - } - uint64_t sum = 0; - for (size_t i = 0; i < count; ++i) { - bool leaf = (out[i].ref & LEAF_BIT) != 0; - if (!out[i].bytes || out[i].bytes > UINT64_MAX - sum || leaf != (height == 1)) return 0; - sum += out[i].bytes; - } - return sum == entry.bytes ? count : 0; -} - -static bool root_height(const Tree *tree, Entry root, unsigned *height) { - *height = 0; - if (!root.bytes) return !root.ref; - if (root.ref & LEAF_BIT) return true; - uint64_t at = root.ref & LOCATION; - if (root.ref & MEMORY_BIT) { - if (at >= tree->count || tree->nodes[at].bytes) return false; - *height = tree->nodes[at].height; - } else { - if (at < DATA || at > tree->end || tree->end - at < 8) return false; - *height = (unsigned)tree->map[at + 6] | (unsigned)tree->map[at + 7] << 8; - } - return *height >= 1 && *height <= HEIGHT_MAX; -} - -/* Visit a range as borrowed extents: 1 done, 0 damaged, -1 stopped by visit. */ -static int visit_range(const Tree *tree, Entry entry, unsigned height, uint64_t offset, - uint64_t length, bool (*visit)(void *, PureSpan), void *context) { - if (!height) { - const unsigned char *bytes = leaf_bytes(tree, entry); - if (!bytes || offset > entry.bytes || length > entry.bytes - offset) return 0; - return visit(context, (PureSpan){bytes + offset, (size_t)length}) ? 1 : -1; - } - Entry kids[FAN_MAX]; - size_t count = children(tree, entry, height, kids); - if (!count) return 0; - uint64_t start = 0; - for (size_t i = 0; i < count && length; ++i) { - uint64_t end = start + kids[i].bytes; - if (offset < end) { - uint64_t inside = offset - start, take = kids[i].bytes - inside; - if (take > length) take = length; - int result = visit_range(tree, kids[i], height - 1, inside, take, visit, context); - if (result != 1) return result; - offset += take; - length -= take; - } - start = end; - } - return length ? 0 : 1; -} - -static bool copy_span(void *context, PureSpan span) { - unsigned char **out = context; - memcpy(*out, span.data, span.size); - *out += span.size; - return true; -} - -static bool copy_range(const Tree *tree, Entry entry, unsigned height, uint64_t offset, - uint64_t length, unsigned char *out) { - return !length || visit_range(tree, entry, height, offset, length, copy_span, &out) == 1; -} - -/* The bytes of a range held by one leaf, or NULL when it spans leaves. */ -static const unsigned char *single_leaf(const Tree *tree, Entry entry, unsigned height, - uint64_t offset, uint64_t length) { - while (height) { - Entry kids[FAN_MAX]; - size_t count = children(tree, entry, height, kids), i = 0; - uint64_t start = 0; - if (!count) return NULL; - while (i + 1 < count && offset >= start + kids[i].bytes) start += kids[i++].bytes; - if (offset - start > kids[i].bytes || length > kids[i].bytes - (offset - start)) return NULL; - entry = kids[i]; - offset -= start; - --height; - } - const unsigned char *bytes = leaf_bytes(tree, entry); - return bytes && offset <= entry.bytes && length <= entry.bytes - offset ? bytes + offset : NULL; -} - -static bool push(List *list, Entry entry) { - if (list->count == list->capacity) { - size_t capacity = list->capacity ? 2 * list->capacity : 8; - Entry *items = realloc(list->items, capacity * sizeof(*items)); - if (!items) return false; - list->items = items; - list->capacity = capacity; - } - list->items[list->count++] = entry; - return true; -} - -static size_t node_bytes(const Node *node) { - return node->bytes ? node->size : node->size * sizeof(Entry); -} - -static bool add_node(Tree *tree, Node node, uint64_t *index) { - tree->bytes += node_bytes(&node); - if (tree->free_head) { - *index = tree->free_head - 1; - tree->free_head = tree->nodes[*index].size; - --tree->free_count; - tree->nodes[*index] = node; - return true; - } - if (tree->count == tree->capacity) { - size_t capacity = tree->capacity ? 2 * tree->capacity : 64; - Node *nodes = realloc(tree->nodes, capacity * sizeof(*nodes)); - if (!nodes) { tree->bytes -= node_bytes(&node); return false; } - tree->nodes = nodes; - tree->capacity = capacity; - } - tree->nodes[tree->count] = node; - *index = tree->count++; - return true; -} - -static void take(Source *source, unsigned char *out, size_t length) { - while (length) { - PureSpan part = source->part[source->index]; - size_t left = part.size - source->offset; - if (!left) { ++source->index; source->offset = 0; continue; } - size_t n = left < length ? left : length; - memcpy(out, part.data + source->offset, n); - out += n; - length -= n; - source->offset += n; - } -} - -/* Balanced leaves: every leaf of a multi-leaf result holds >= LEAF_MAX / 2. */ -static bool chunk_bytes(Tree *tree, Source *source, uint64_t total, List *out) { - uint64_t pieces = (total + LEAF_MAX - 1) / LEAF_MAX; - for (uint64_t i = 0; i < pieces; ++i) { - size_t size = (size_t)(total / pieces + (i < total % pieces)); - unsigned char *bytes = malloc(size); - uint64_t index; - if (!bytes) return false; - take(source, bytes, size); - if (!add_node(tree, (Node){.bytes = bytes, .size = size}, &index)) { free(bytes); return false; } - if (!push(out, (Entry){LEAF_BIT | MEMORY_BIT | index, size})) return false; - } - return true; -} - -static bool chunk_entries(Tree *tree, const Entry *items, size_t count, unsigned height, List *out) { - size_t pieces = (count + FAN_MAX - 1) / FAN_MAX; - for (size_t i = 0, at = 0; i < pieces; ++i) { - size_t size = count / pieces + (i < count % pieces); - Entry *kids = malloc(size * sizeof(*kids)); - uint64_t sum = 0, index; - if (!kids) return false; - memcpy(kids, items + at, size * sizeof(*kids)); - for (size_t k = 0; k < size; ++k) sum += kids[k].bytes; - if (!add_node(tree, (Node){.children = kids, .size = size, .height = height}, &index)) { - free(kids); - return false; - } - if (!push(out, (Entry){MEMORY_BIT | index, sum})) return false; - at += size; - } - return true; -} - -/* Only nodes created by this edit can be underfull; stored siblings are left as they are. */ -static bool underfull(const Tree *tree, Entry entry, unsigned height) { - if (!(entry.ref & MEMORY_BIT)) return false; - return height ? tree->nodes[entry.ref & LOCATION].size < FAN_MIN : entry.bytes < LEAF_MIN; -} - -static bool replace(List *list, size_t at, size_t removed, const List *with) { - size_t count = list->count - removed + with->count; - if (count > list->capacity) { - Entry *items = realloc(list->items, count * sizeof(*items)); - if (!items) return false; - list->items = items; - list->capacity = count; - } - memmove(list->items + at + with->count, list->items + at + removed, - (list->count - at - removed) * sizeof(*list->items)); - memcpy(list->items + at, with->items, with->count * sizeof(*list->items)); - list->count = count; - return true; -} - -static bool merge(Tree *tree, Entry a, Entry b, unsigned height, List *out); - -/* Merge each new underfull node with a neighbour. A merge removes a node or - * yields two nodes of at least half capacity, so the loop terminates. */ -static bool repair(Tree *tree, List *all, unsigned height) { - bool ok = true; - for (size_t i = 0; ok && all->count > 1 && i < all->count;) { - if (!underfull(tree, all->items[i], height)) { ++i; continue; } - size_t at = i + 1 < all->count ? i : i - 1; - List merged = {0}; - ok = merge(tree, all->items[at], all->items[at + 1], height, &merged) && - replace(all, at, 2, &merged); - free(merged.items); - i = at; - } - return ok; -} - -static bool merge(Tree *tree, Entry a, Entry b, unsigned height, List *out) { - if (!height) { - const unsigned char *x = leaf_bytes(tree, a), *y = leaf_bytes(tree, b); - if (!x || !y) return false; - Source source = {.part = {{x, (size_t)a.bytes}, {y, (size_t)b.bytes}}}; - return chunk_bytes(tree, &source, a.bytes + b.bytes, out); - } - Entry kids[FAN_MAX]; - List all = {0}; - size_t count = children(tree, a, height, kids); - bool ok = count != 0; - for (size_t i = 0; ok && i < count; ++i) ok = push(&all, kids[i]); - count = ok ? children(tree, b, height, kids) : 0; - ok = count != 0; - for (size_t i = 0; ok && i < count; ++i) ok = push(&all, kids[i]); - /* A small child carried in from a one-child parent is repaired here too. */ - ok = ok && repair(tree, &all, height - 1) && chunk_entries(tree, all.items, all.count, height, out); - free(all.items); - return ok; -} - -/* Replace [offset, offset + remove) of entry with insert. Appends the - * replacement nodes, all at entry's height, to out (possibly none). */ -static bool edit(Tree *tree, Entry entry, unsigned height, uint64_t offset, uint64_t remove, - PureSpan insert, List *out) { - if (offset > entry.bytes || remove > entry.bytes - offset) return false; - if (!height) { - const unsigned char *bytes = leaf_bytes(tree, entry); - if (!bytes) return false; - Source source = {.part = {{bytes, (size_t)offset}, insert, - {bytes + offset + remove, (size_t)(entry.bytes - offset - remove)}}}; - return chunk_bytes(tree, &source, entry.bytes - remove + insert.size, out); - } - Entry kids[FAN_MAX]; - size_t count = children(tree, entry, height, kids), first = count, final = count; - if (!count) return false; - uint64_t last = remove ? offset + remove - 1 : offset, start = 0, first_start = 0, final_start = 0; - for (size_t i = 0; i < count; ++i) { - uint64_t end = start + kids[i].bytes; - bool tail = i + 1 == count; - if (first == count && (offset < end || tail)) { first = i; first_start = start; } - if (first != count && (last < end || tail)) { final = i; final_start = start; break; } - start = end; - } - List middle = {0}, all = {0}; - bool ok = first == final ? - edit(tree, kids[first], height - 1, offset - first_start, remove, insert, &middle) : - edit(tree, kids[first], height - 1, offset - first_start, - first_start + kids[first].bytes - offset, insert, &middle) && - edit(tree, kids[final], height - 1, 0, offset + remove - final_start, (PureSpan){0}, &middle); - for (size_t i = 0; ok && i < first; ++i) ok = push(&all, kids[i]); - for (size_t i = 0; ok && i < middle.count; ++i) ok = push(&all, middle.items[i]); - for (size_t i = final + 1; ok && i < count; ++i) ok = push(&all, kids[i]); - free(middle.items); - ok = ok && repair(tree, &all, height - 1) && chunk_entries(tree, all.items, all.count, height, out); - free(all.items); - return ok; -} - -/* The new root of root with [offset, offset + remove) replaced by insert. */ -static bool rebuild(Tree *tree, Entry root, uint64_t offset, uint64_t remove, - PureSpan insert, Entry *out) { - unsigned height; - List list = {0}; - bool ok = root_height(tree, root, &height); - if (ok && !root.bytes) { - Source source = {.part = {insert}}; - ok = !offset && !remove && chunk_bytes(tree, &source, insert.size, &list); - } else if (ok) ok = edit(tree, root, height, offset, remove, insert, &list); - while (ok && list.count > 1) { - List up = {0}; - ok = height < HEIGHT_MAX && chunk_entries(tree, list.items, list.count, ++height, &up); - free(list.items); - list = up; - } - Entry result = ok && list.count ? list.items[0] : (Entry){0}; - free(list.items); - while (ok && height && result.bytes) { - Entry kids[FAN_MAX]; - size_t count = children(tree, result, height, kids); - if (count != 1) { ok = count > 1; break; } - result = kids[0]; - --height; - } - *out = result; - return ok; -} - -static bool flush(Writer *writer) { - if (writer->used && !writer->failed) { - digest_add(&writer->digest, writer->buffer, writer->used); - if (!write_at(writer->fd, writer->buffer, writer->used, writer->at - writer->used)) writer->failed = true; - } - writer->used = 0; - return !writer->failed; -} - -/* Zeros from..to; a failure only costs speed. */ -static void zero_fill(int fd, uint64_t from, uint64_t to) { - static const unsigned char zeros[BUFFER]; - while (from < to) { - size_t n = to - from < BUFFER ? (size_t)(to - from) : BUFFER; - if (!write_at(fd, zeros, n, from)) return; - from += n; - } -} - -/* Where the written zeros after a published end should reach. */ -static uint64_t tail_end(uint64_t end) { - return end + ((end - DATA) / 8 > TAIL_STEP ? (end - DATA) / 8 : TAIL_STEP); -} - -static void put_bytes(Writer *writer, const unsigned char *bytes, size_t length) { - while (length && !writer->failed) { - if (writer->used == BUFFER && !flush(writer)) return; - size_t n = BUFFER - writer->used < length ? BUFFER - writer->used : length; - memcpy(writer->buffer + writer->used, bytes, n); - writer->used += n; - writer->at += n; - bytes += n; - length -= n; - } -} - -static uint64_t put_branch(Writer *writer, const Entry *kids, size_t count, unsigned height) { - static const unsigned char zero[8]; - put_bytes(writer, zero, (size_t)((8 - writer->at % 8) % 8)); - uint64_t at = writer->at; - unsigned char header[8]; - memcpy(header, branch_magic, sizeof(branch_magic)); - header[4] = (unsigned char)count; - header[5] = (unsigned char)(count >> 8); - header[6] = (unsigned char)height; - header[7] = (unsigned char)(height >> 8); - put_bytes(writer, header, sizeof(header)); - for (size_t i = 0; i < count; ++i) { - unsigned char pair[16]; - encode64(pair, kids[i].ref); - encode64(pair + 8, kids[i].bytes); - put_bytes(writer, pair, sizeof(pair)); - } - return at; -} - -/* Append a subtree's arena nodes, children first, and set entry to the - * stored reference. The arena is not changed, so a failed commit leaves a - * draft intact. With relocate, stored nodes are copied too (compaction, or a - * draft whose base file was replaced by compaction). */ -static bool write_subtree(const Tree *tree, Writer *writer, Entry *entry, unsigned height, - bool relocate) { - if (!entry->bytes || (!(entry->ref & MEMORY_BIT) && !relocate)) return true; - if (!height) { - const unsigned char *bytes = leaf_bytes(tree, *entry); - if (!bytes) return false; - entry->ref = LEAF_BIT | writer->at; - put_bytes(writer, bytes, (size_t)entry->bytes); - return !writer->failed; - } - Entry kids[FAN_MAX]; - size_t count = children(tree, *entry, height, kids); - if (!count) return false; - for (size_t i = 0; i < count; ++i) - if (!write_subtree(tree, writer, &kids[i], height - 1, relocate)) return false; - entry->ref = put_branch(writer, kids, count, height); - return !writer->failed; -} - -static bool write_root(const Tree *tree, Writer *writer, Entry *root, bool relocate) { - unsigned height; - return root_height(tree, *root, &height) && write_subtree(tree, writer, root, height, relocate); -} - -static bool read_record(const Tree *tree, Entry catalog, uint64_t index, - unsigned char record[RECORD], Entry *document, int64_t *revision) { - unsigned height; - if (!root_height(tree, catalog, &height) || - !copy_range(tree, catalog, height, index * RECORD, RECORD, record)) return false; - *document = (Entry){decode64(record + KEY), decode64(record + KEY + 8)}; - uint64_t stored = decode64(record + KEY + 16); - *revision = (int64_t)stored; - return stored && stored <= (uint64_t)INT64_MAX && !document->ref == !document->bytes && - document->bytes <= PURE_OBJECT_LIMIT; -} - -/* Binary search over records ordered by their zero-padded key bytes. */ -static bool find(const Tree *tree, Entry catalog, const char *key, uint64_t *index, - bool *found, Entry *document, int64_t *revision) { - unsigned char wanted[KEY] = {0}, record[RECORD]; - memcpy(wanted, key, strlen(key)); - uint64_t low = 0, high = catalog.bytes / RECORD; - *found = false; - *document = (Entry){0}; - *revision = 0; - while (low < high) { - uint64_t middle = low + (high - low) / 2; - if (!read_record(tree, catalog, middle, record, document, revision)) return false; - int order = memcmp(wanted, record, KEY); - if (!order) { *index = middle; *found = true; return true; } - if (order < 0) high = middle; else low = middle + 1; - } - *document = (Entry){0}; - *revision = 0; - *index = low; - return true; -} - -static int create_temporary(int directory, char name[96]) { - for (unsigned attempt = 0; attempt < 64; ++attempt) { - if (temporary_sequence == UINT64_MAX) break; - snprintf(name, 96, ".pending-%ld-%" PRIu64, (long)getpid(), ++temporary_sequence); - int fd = openat(directory, name, O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600); - if (fd >= 0) return fd; - if (errno != EEXIST) break; - } - name[0] = 0; /* Never remove an existing file on a name collision. */ - return -1; -} - -/* Publish a fresh file with both slots pointing at head. It is synced whole - * before it is renamed into place, so its head names no commit bytes. */ -static bool install(int directory, char temporary[96], int fd, Head *head) { - unsigned char slot[SLOT]; - struct stat file; - head->start = head->end; - head->check = digest_end(&(Digest){0}); - encode_head(slot, head_magic, head); - zero_fill(fd, head->end, tail_end(head->end)); - bool ok = write_at(fd, slot, SLOT, 0) && write_at(fd, slot, SLOT, BLOCK) && sync_fd(fd) && !fstat(fd, &file); - if (close(fd)) ok = false; - ok = ok && !renameat(directory, temporary, directory, tree_name); - if (ok) temporary[0] = 0; - ok = ok && sync_fd(directory); - if (ok) remember(&file, head, DURABLE); - return ok; -} - -/* Rewrite only reachable nodes. Open readers keep their mapping of the - * previous file. A failure leaves the published file untouched. */ -static void compact(int directory, int source, const struct stat *file, const Head *head) { - Tree tree; - if (!map_tree(source, file, head, &tree)) return; - char temporary[96]; - int fd = create_temporary(directory, temporary); - Writer *writer = fd >= 0 ? malloc(sizeof(*writer)) : NULL; - uint64_t count = head->catalog_bytes / RECORD; - unsigned char *records = count ? malloc((size_t)(count * RECORD)) : NULL; - Entry catalog = {head->catalog, head->catalog_bytes}; - unsigned height; - bool ok = writer && (!count || records) && root_height(&tree, catalog, &height) && - (!count || copy_range(&tree, catalog, height, 0, count * RECORD, records)); - if (writer) *writer = (Writer){.fd = fd, .at = DATA}; - for (uint64_t i = 0; ok && i < count; ++i) { - unsigned char *record = records + i * RECORD; - Entry document = {decode64(record + KEY), decode64(record + KEY + 8)}; - unsigned document_height; - ok = root_height(&tree, document, &document_height) && - write_subtree(&tree, writer, &document, document_height, true); - encode64(record + KEY, document.ref); - } - Entry fresh = {0}; - ok = ok && rebuild(&tree, (Entry){0}, 0, 0, (PureSpan){records, (size_t)(count * RECORD)}, &fresh) && - write_root(&tree, writer, &fresh, false) && flush(writer); - if (ok) { - Head compacted = {head->seq + 1, writer->at, fresh.ref, fresh.bytes, writer->at - DATA, 0, 0}; - ok = install(directory, temporary, fd, &compacted); - fd = -1; - } - if (fd >= 0) close(fd); - if (*temporary) unlinkat(directory, temporary, 0); - free(records); - free(writer); - release_tree(&tree); -} - -static bool lock_writer(int lock, bool wait) { - struct flock hold = {.l_type = F_WRLCK, .l_whence = SEEK_SET, .l_start = 0, .l_len = 0}; - int rc; - do { rc = fcntl(lock, wait ? F_SETLKW : F_SETLK, &hold); } while (rc && wait && errno == EINTR); - return rc == 0; -} - -PureResult pure_store_init(const char *path) { - if (!path || !*path) return PURE_STORE_ERROR; - bool created = mkdir(path, 0700) == 0; - if (!created && errno != EEXIST) return PURE_STORE_ERROR; - int directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); - if (directory < 0) return PURE_STORE_ERROR; /* Never reinterpret an old DB file. */ - pthread_mutex_lock(&writers); - int lock = openat(directory, ".writer.lock", O_RDWR | O_CREAT | O_NOFOLLOW | O_CLOEXEC, 0600); - struct stat info; - bool ok = lock >= 0 && fstat(lock, &info) == 0 && S_ISREG(info.st_mode) && lock_writer(lock, true); - if (ok) { - Head head; - struct stat file; - int fd = openat(directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - if (fd >= 0) { - ok = read_head(fd, &file, true, &head, NULL); /* An unreadable file is never replaced. */ - close(fd); - } else if (errno != ENOENT) ok = false; - else { - char temporary[96]; - int created_fd = create_temporary(directory, temporary); - head = (Head){.seq = 1, .end = DATA}; - ok = created_fd >= 0 && install(directory, temporary, created_fd, &head); - if (*temporary) unlinkat(directory, temporary, 0); - } - } - ok = ok && sync_fd(lock) && sync_fd(directory); - if (lock >= 0) close(lock); - pthread_mutex_unlock(&writers); - close(directory); - if (ok && created) { - char *parent = strdup(path); - if (!parent) return PURE_STORE_ERROR; - size_t length = strlen(parent); - while (length > 1 && parent[length - 1] == '/') parent[--length] = 0; - char *slash = strrchr(parent, '/'); - if (!slash) strcpy(parent, "."); - else if (slash == parent) parent[1] = 0; - else *slash = 0; - int fd = open(parent, O_RDONLY | O_DIRECTORY | O_CLOEXEC); - free(parent); - ok = fd >= 0 && sync_fd(fd); - if (fd >= 0) close(fd); - } - return ok ? PURE_OK : PURE_STORE_ERROR; -} - -void pure_store_release(PureStoredView *view) { - if (view->mapped_size) munmap(view->mapping, view->mapped_size); - *view = (PureStoredView){0}; -} - -PureResult pure_store_select(const char *path, const char *key, size_t offset, - size_t length, bool whole, size_t minimum, - size_t available, PureStoredView *out) { - *out = (PureStoredView){0}; - if (!valid_key(key)) return PURE_STORE_ERROR; - int directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); - if (directory < 0) return PURE_STORE_ERROR; - int fd = openat(directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - close(directory); - if (fd < 0) return PURE_STORE_ERROR; - Tree tree = {0}; - Head head; - struct stat file; - Entry document; - uint64_t index; - int64_t revision; - bool found; - unsigned height; - PureResult result = PURE_STORE_ERROR; - if (!read_head(fd, &file, false, &head, NULL) || !map_tree(fd, &file, &head, &tree) || - !find(&tree, (Entry){head.catalog, head.catalog_bytes}, key, &index, &found, - &document, &revision)) goto finish; - if (!found) { result = PURE_NOT_FOUND; goto finish; } - size_t size = (size_t)document.bytes; - if (size < minimum || offset > size) { result = PURE_LIMIT; goto finish; } - if (whole) length = size - offset; - if (length > size - offset) { result = PURE_LIMIT; goto finish; } - if (length > available) { result = PURE_BUDGET; goto finish; } - if (length) { - if (!root_height(&tree, document, &height)) goto finish; - const unsigned char *direct = single_leaf(&tree, document, height, offset, length); - void *mapping; - size_t mapped_size, prefix = 0; - if (direct) { - /* One extent: map it in place, as the flat store does. */ - long page = sysconf(_SC_PAGESIZE); - if (page <= 0) goto finish; - size_t start = (size_t)(direct - tree.map), aligned = start - start % (size_t)page; - prefix = start - aligned; - mapped_size = prefix + length; - mapping = mmap(NULL, mapped_size, PROT_READ, MAP_PRIVATE, fd, (off_t)aligned); - if (mapping == MAP_FAILED) goto finish; - } else { - /* A contiguous view of several extents is assembled for this API. */ - mapped_size = length; - mapping = mmap(NULL, mapped_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (mapping == MAP_FAILED) goto finish; - if (!copy_range(&tree, document, height, offset, length, mapping) || - mprotect(mapping, mapped_size, PROT_READ)) { - munmap(mapping, mapped_size); - goto finish; - } - } - out->mapping = mapping; - out->mapped_size = mapped_size; - out->body = (PureSpan){(const unsigned char *)mapping + prefix, length}; - } - out->revision = revision; - out->object_size = size; - result = PURE_OK; -finish: - release_tree(&tree); - close(fd); - return result; -} - -PureResult pure_store_get(const char *path, const char *key, size_t available, PureStoredView *out) { - return pure_store_select(path, key, 0, 0, true, 0, available, out); -} - -PureResult pure_store_get_range(const char *path, const char *key, size_t offset, - size_t length, size_t available, PureStoredView *out) { - return pure_store_select(path, key, offset, length, false, 0, available, out); -} - -/* A locked writer over the current published file. */ -typedef struct { - int directory, lock, fd; - Head head; - uint64_t slots[2]; /* the sequences of the slots' heads */ - Tree tree; - struct stat file; - bool turn; /* holds this process's writer turn */ -} Session; - -/* Keep written zeros after the published end, where commits and their - * unpublished leftovers go. A sync of blocks the file already has writes - * only them; a new size or new blocks add a journal commit (on an SD card - * 2.4 ms against 6 ms per commit). The zeros become durable with the next - * commit's sync. */ -static void prepare_tail(Session *session) { - uint64_t size = (uint64_t)session->file.st_size, end = session->head.end; - if (size < end + TAIL_MIN) zero_fill(session->fd, size > end ? size : end, tail_end(end)); -} - -static void close_session(Session *session) { - release_tree(&session->tree); - if (session->fd >= 0) close(session->fd); - if (session->lock >= 0) close(session->lock); - if (session->directory >= 0) close(session->directory); - if (session->turn) pthread_mutex_unlock(&writers); - session->turn = false; -} - -static PureResult open_session(const char *path, Session *session) { - *session = (Session){.directory = -1, .lock = -1, .fd = -1}; - pthread_mutex_lock(&writers); /* another thread of this process waits, not fails */ - session->turn = true; - session->directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); - if (session->directory < 0) return PURE_STORE_ERROR; - session->lock = openat(session->directory, ".writer.lock", O_RDWR | O_NOFOLLOW | O_CLOEXEC); - struct stat info; - if (session->lock < 0 || fstat(session->lock, &info) || !S_ISREG(info.st_mode)) return PURE_STORE_ERROR; - if (!lock_writer(session->lock, false)) - return errno == EACCES || errno == EAGAIN ? PURE_BUSY : PURE_STORE_ERROR; - session->fd = openat(session->directory, tree_name, O_RDWR | O_NOFOLLOW | O_CLOEXEC); - if (session->fd < 0 || !read_head(session->fd, &session->file, true, &session->head, session->slots)) - return PURE_STORE_ERROR; - /* A head this process did not make durable may be the last write of a - * writer that died before its sync: make it durable before building on it. */ - if (recall(&session->file, &session->head) != DURABLE) { - if (!sync_data(session->fd)) return PURE_STORE_ERROR; - remember(&session->file, &session->head, DURABLE); - } - prepare_tail(session); - return map_tree(session->fd, &session->file, &session->head, &session->tree) ? PURE_OK : PURE_STORE_ERROR; -} - -/* A commit's bytes start after room for its record at the published end. */ -static uint64_t commit_start(const Session *session) { return session->head.end + SLOT; } - -/* The documents' new nodes are already written at writer. Record each in - * the catalog (revisions come in as the base and leave as the new one), then - * write the head as the record before them and sync once: every document - * becomes visible at once, or none does. Readers of this process see the - * head only once it is durable. */ -#ifdef PURE_STORE_TEST -static unsigned long publications; -unsigned long pure_tree_test_publications(void) { return publications; } -#endif - -static PureResult publish(Session *session, Writer *writer, size_t count, const char *const *keys, - const Entry *documents, int64_t *revisions) { -#ifdef PURE_STORE_TEST - ++publications; -#endif - Entry catalog = {session->head.catalog, session->head.catalog_bytes}; - bool written = true; - for (size_t i = 0; written && i < count; ++i) { - unsigned char record[RECORD] = {0}; - uint64_t index; - bool found; - Entry current, updated; - int64_t stored; - memcpy(record, keys[i], strlen(keys[i])); - encode64(record + KEY, documents[i].ref); - encode64(record + KEY + 8, documents[i].bytes); - encode64(record + KEY + 16, (uint64_t)revisions[i] + 1); - written = find(&session->tree, catalog, keys[i], &index, &found, ¤t, &stored) && - rebuild(&session->tree, catalog, index * RECORD, found ? RECORD : 0, - (PureSpan){record, RECORD}, &updated); - catalog = updated; - } - written = written && write_root(&session->tree, writer, &catalog, false) && flush(writer); - /* Unpublished bytes stay in the tail; the next commit overwrites them. */ - if (!written || fault(PURE_AFTER_HEADER) || fault(PURE_BEFORE_FILE_SYNC) || fault(PURE_BEFORE_RENAME)) - return PURE_STORE_ERROR; - Head published = {session->head.seq + 1, writer->at, catalog.ref, catalog.bytes, session->head.live, - commit_start(session), digest_end(&writer->digest)}; - unsigned char record[SLOT], slot[SLOT]; - encode_head(record, record_magic, &published); - encode_head(slot, head_magic, &published); - /* Every few commits the head goes into the older slot as well: one more - * range for that sync, and fewer records for a reader to follow. */ - uint64_t newer = session->slots[0] > session->slots[1] ? session->slots[0] : session->slots[1]; - unsigned older = session->slots[0] > session->slots[1]; - bool checkpoint = published.seq - newer >= CHECKPOINT; - for (size_t i = 0; i < count; ++i) ++revisions[i]; /* Past this point the head may already be visible. */ - remember(&session->file, &published, PUBLISHING); - bool durable = write_at(session->fd, record, SLOT, published.start - SLOT) && - (!checkpoint || write_at(session->fd, slot, SLOT, older * BLOCK)) && - !fault(PURE_BEFORE_DIR_SYNC) && sync_data(session->fd); - remember(&session->file, &published, durable ? DURABLE : CHECKED); - if (!durable) return PURE_UNCERTAIN; - if (published.end - DATA > 4 * published.live + COMPACT_SLACK && !fstat(session->fd, &session->file)) - compact(session->directory, session->fd, &session->file, &published); - return PURE_OK; -} - -static PureResult mutate(const char *path, const char *key, int64_t expected, - PureSpan body, bool splice, size_t offset, size_t remove, - size_t max_size, int64_t *revision) { - *revision = 0; - if (!valid_key(key) || (body.size && !body.data)) return PURE_STORE_ERROR; - if (body.size > PURE_OBJECT_LIMIT || max_size > PURE_OBJECT_LIMIT || expected < 0) return PURE_LIMIT; - Session session; - Writer *writer = NULL; - Entry document, next; - uint64_t index; - bool found; - PureResult result = open_session(path, &session); - if (result != PURE_OK) goto finish; - result = PURE_STORE_ERROR; - if (!find(&session.tree, (Entry){session.head.catalog, session.head.catalog_bytes}, key, - &index, &found, &document, revision)) goto finish; - if (*revision != expected) { result = PURE_CONFLICT; goto finish; } - if (*revision == INT64_MAX) { result = PURE_LIMIT; goto finish; } - uint64_t old_size = document.bytes; - if (splice && (offset > old_size || remove > old_size - offset || - old_size - remove > max_size || - body.size > max_size - (old_size - remove))) { result = PURE_LIMIT; goto finish; } - writer = malloc(sizeof(*writer)); - if (!writer || !rebuild(&session.tree, splice ? document : (Entry){0}, splice ? offset : 0, - splice ? remove : 0, body, &next)) goto finish; - *writer = (Writer){.fd = session.fd, .at = commit_start(&session)}; - if (!write_root(&session.tree, writer, &next, false)) goto finish; - result = publish(&session, writer, 1, &key, &next, revision); -finish: - free(writer); - close_session(&session); - return result; -} - -/* Keys come from the catalog in ascending byte order. Runtime values are not - * documents and are left out. */ -PureResult pure_store_keys(const char *path, bool (*visit)(void *, const char *key), - void *context) { - int directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); - if (directory < 0) return PURE_STORE_ERROR; - int fd = openat(directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - close(directory); - if (fd < 0) return PURE_STORE_ERROR; - Tree tree = {0}; - Head head; - struct stat file; - PureResult result = PURE_STORE_ERROR; - if (read_head(fd, &file, false, &head, NULL) && map_tree(fd, &file, &head, &tree)) { - Entry catalog = {head.catalog, head.catalog_bytes}, document; - unsigned char record[RECORD]; - char key[KEY + 1] = {0}; - int64_t revision; - result = PURE_OK; - for (uint64_t i = 0; i < catalog.bytes / RECORD; ++i) { - if (!read_record(&tree, catalog, i, record, &document, &revision)) { result = PURE_STORE_ERROR; break; } - memcpy(key, record, KEY); - if (runtime_key(key)) continue; - if (!valid_key(key)) { result = PURE_STORE_ERROR; break; } - if (!visit(context, key)) break; - } - } - release_tree(&tree); - close(fd); - return result; -} - -/* A store kept open by one owner thread: its current file and a mapping of - * its newest version, shared by the drafts opened through it. */ -struct PureStore { - char *path; - int directory, fd; - dev_t device; - ino_t inode; - Head head; - Mapping *mapping; - bool pinned; /* drafts open at the version of the last check */ -}; - -struct PureDraft { - PureStore *store; /* the handle it was opened through, or NULL */ - char *path; /* owned unless opened through a store handle */ - char key[KEY + 1]; - Tree tree; /* the base file's mapping plus this draft's node arena */ - dev_t device; /* identity of the base file; compaction replaces it */ - ino_t inode; - Entry root; - int64_t revision; - size_t live; /* arena bytes reachable at the last collection */ - bool held; /* a transaction still reads earlier versions */ -}; - -/* Rebase the draft on the committed version of its key. */ -static PureResult load_draft(PureDraft *draft) { - int directory = open(draft->path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); - if (directory < 0) return PURE_STORE_ERROR; - int fd = openat(directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - close(directory); - if (fd < 0) return PURE_STORE_ERROR; - Tree tree = {0}; - Head head; - struct stat info; - Entry document; - uint64_t index; - int64_t revision; - bool found; - bool ok = read_head(fd, &info, false, &head, NULL) && map_tree(fd, &info, &head, &tree) && - find(&tree, (Entry){head.catalog, head.catalog_bytes}, draft->key, &index, &found, - &document, &revision); - close(fd); - if (!ok) { release_tree(&tree); return PURE_STORE_ERROR; } - release_tree(&draft->tree); - draft->tree = tree; - draft->device = info.st_dev; - draft->inode = info.st_ino; - draft->root = document; - draft->revision = revision; - draft->live = 0; - return PURE_OK; -} - -PureResult pure_draft_open(const char *path, const char *key, PureDraft **out) { - *out = NULL; - if (!path || !(valid_key(key) || runtime_key(key))) return PURE_INVALID; - PureDraft *draft = calloc(1, sizeof(*draft)); - if (!draft || !(draft->path = strdup(path))) { free(draft); return PURE_ALLOCATION; } - memcpy(draft->key, key, strlen(key) + 1); - PureResult result = load_draft(draft); - if (result != PURE_OK) { pure_draft_close(draft); return result; } - *out = draft; - return PURE_OK; -} - -PureResult pure_state_open(const char *path, const char *name, PureDraft **out) { - char key[KEY + 2]; - *out = NULL; - if (!valid_key(name) || strlen(name) >= KEY) return PURE_INVALID; - snprintf(key, sizeof(key), ".%s", name); - return pure_draft_open(path, key, out); -} - -PureDraft *pure_draft_new(void) { - return calloc(1, sizeof(PureDraft)); /* no path: nothing mapped, nothing to commit */ -} - -PureDraft *pure_draft_freeze(const PureDraft *draft) { - if (!draft->tree.shared || (draft->root.ref & MEMORY_BIT)) return NULL; - PureDraft *frozen = calloc(1, sizeof(*frozen)); - if (!frozen) return NULL; - Mapping *mapping = draft->tree.shared; - __atomic_add_fetch(&mapping->refs, 1, __ATOMIC_RELAXED); - *frozen = (PureDraft){.tree = {.map = mapping->map, .end = mapping->end, .shared = mapping}, - .device = draft->device, .inode = draft->inode, .root = draft->root, - .revision = draft->revision}; - memcpy(frozen->key, draft->key, sizeof(frozen->key)); - return frozen; -} - -void pure_draft_close(PureDraft *draft) { - if (!draft) return; - release_tree(&draft->tree); - if (!draft->store) free(draft->path); - free(draft); -} - -/* Bring the handle to the newest published version: an unchanged store costs - * one system call (the mapping shows the head slots); a new commit maps - * again, a compaction reopens. */ -static PureResult refresh(PureStore *store) { - struct stat info; - if (fstatat(store->directory, tree_name, &info, AT_SYMLINK_NOFOLLOW) || !S_ISREG(info.st_mode)) - return PURE_STORE_ERROR; - if (store->fd < 0 || info.st_dev != store->device || info.st_ino != store->inode) { - int fd = openat(store->directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - if (fd < 0 || fstat(fd, &info)) { if (fd >= 0) close(fd); return PURE_STORE_ERROR; } - if (store->fd >= 0) close(store->fd); - store->fd = fd; - store->device = info.st_dev; - store->inode = info.st_ino; - store->head = (Head){0}; - } - if (store->head.seq) { - /* The next commit's record follows the head's bytes; a slot shows a - * newer head only for a commit of the earlier format. */ - const unsigned char *map = store->mapping->map; - unsigned char next[16]; - bool newer = decode64(map + 8) > store->head.seq || decode64(map + BLOCK + 8) > store->head.seq || - (read_at(store->fd, next, sizeof(next), store->head.end) && !memcmp(next, record_magic, 8) && - decode64(next + 8) == store->head.seq + 1); - if (!newer) return PURE_OK; - } - Head head; - if (!read_head(store->fd, &info, false, &head, NULL)) return PURE_STORE_ERROR; - if (store->mapping && head.seq == store->head.seq && head.check == store->head.check) return PURE_OK; - Tree tree; - Mapping *mapping = malloc(sizeof(*mapping)); - if (!mapping) return PURE_ALLOCATION; - if (!map_tree(store->fd, &info, &head, &tree)) { free(mapping); return PURE_STORE_ERROR; } - *mapping = (Mapping){tree.map, tree.end, 1}; - release_mapping(store->mapping); - store->mapping = mapping; - store->head = head; - return PURE_OK; -} - -/* Base a draft on the handle's newest version of its key, sharing the mapping. - * On failure the draft is unchanged. */ -static PureResult rebase(PureDraft *draft) { - PureStore *store = draft->store; - Mapping *mapping = store->mapping; - Tree tree = {.map = mapping->map, .end = mapping->end, .shared = mapping}; - Entry document; - uint64_t index; - int64_t revision; - bool found; - if (!find(&tree, (Entry){store->head.catalog, store->head.catalog_bytes}, draft->key, &index, &found, &document, - &revision)) return PURE_STORE_ERROR; - __atomic_add_fetch(&mapping->refs, 1, __ATOMIC_RELAXED); - release_tree(&draft->tree); - draft->tree = tree; - draft->device = store->device; - draft->inode = store->inode; - draft->root = document; - draft->revision = revision; - draft->live = 0; - return PURE_OK; -} - -PureResult pure_store_open(const char *path, PureStore **out) { - *out = NULL; - PureStore *store = calloc(1, sizeof(*store)); - if (!store || !path || !(store->path = strdup(path))) { free(store); return PURE_ALLOCATION; } - store->fd = -1; - store->directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); - PureResult result = store->directory < 0 ? PURE_STORE_ERROR : refresh(store); - if (result != PURE_OK) { pure_store_close(store); return result; } - *out = store; - return PURE_OK; -} - -void pure_store_close(PureStore *store) { - if (!store) return; - release_mapping(store->mapping); - if (store->fd >= 0) close(store->fd); - if (store->directory >= 0) close(store->directory); - free(store->path); - free(store); -} - -PureMapping *pure_draft_mapping(const PureDraft *draft) { - Mapping *mapping = draft->tree.shared; - if (mapping) __atomic_add_fetch(&mapping->refs, 1, __ATOMIC_RELAXED); - return mapping; -} - -bool pure_mapping_holds(const PureMapping *mapping, PureSpan span) { - uintptr_t base = (uintptr_t)mapping->map, at = (uintptr_t)span.data; - return !span.size || (span.data && at >= base && at - base <= mapping->end && span.size <= mapping->end - (at - base)); -} - -void pure_mapping_release(PureMapping *mapping) { release_mapping(mapping); } - -PureResult pure_store_pin(PureStore *store, bool pinned) { - store->pinned = false; - PureResult result = pinned ? refresh(store) : PURE_OK; - store->pinned = pinned && result == PURE_OK; - return result; -} - -static PureResult open_through(PureStore *store, const char *key, PureDraft **out) { - *out = NULL; - PureResult result = store->pinned ? PURE_OK : refresh(store); - if (result != PURE_OK) return result; - PureDraft *draft = calloc(1, sizeof(*draft)); - if (!draft) return PURE_ALLOCATION; - *draft = (PureDraft){.store = store, .path = store->path}; - memcpy(draft->key, key, strlen(key) + 1); - result = rebase(draft); - if (result != PURE_OK) { free(draft); return result; } - *out = draft; - return PURE_OK; -} - -PureResult pure_store_draft(PureStore *store, const char *key, PureDraft **out) { - *out = NULL; - return store && valid_key(key) ? open_through(store, key, out) : PURE_INVALID; -} - -PureResult pure_store_state(PureStore *store, const char *name, PureDraft **out) { - char key[KEY + 2]; - *out = NULL; - if (!store || !valid_key(name) || strlen(name) >= KEY) return PURE_INVALID; - snprintf(key, sizeof(key), ".%s", name); - return open_through(store, key, out); -} - -size_t pure_draft_size(const PureDraft *draft) { return (size_t)draft->root.bytes; } - -int64_t pure_draft_revision(const PureDraft *draft) { return draft->revision; } - -static size_t mark(const Tree *tree, Entry entry, unsigned char *marks) { - if (!(entry.ref & MEMORY_BIT)) return 0; - size_t at = (size_t)(entry.ref & LOCATION); - if (marks[at]) return 0; - marks[at] = 1; - const Node *node = &tree->nodes[at]; - size_t bytes = node_bytes(node); - if (!node->bytes) - for (size_t i = 0; i < node->size; ++i) bytes += mark(tree, node->children[i], marks); - return bytes; -} - -/* Every edit copies a leaf and path, so reclaim arena nodes the root no - * longer reaches once they outweigh the reachable ones (and 32 KiB). */ -static void collect(PureDraft *draft) { - Tree *tree = &draft->tree; - if (draft->held) return; - size_t allowance = draft->live > 32768 ? draft->live : 32768; - if (tree->bytes <= draft->live || tree->bytes - draft->live <= allowance) return; - unsigned char *marks = calloc(tree->count, 1); - if (!marks) return; - draft->live = mark(tree, draft->root, marks); - for (size_t i = 0; i < tree->count; ++i) { - Node *node = &tree->nodes[i]; - if (marks[i] || node->height == FREE) continue; - tree->bytes -= node_bytes(node); - free(node->bytes); - free(node->children); - *node = (Node){.size = tree->free_head, .height = FREE}; - tree->free_head = i + 1; - ++tree->free_count; - } - free(marks); -} - -PureResult pure_draft_splice(PureDraft *draft, size_t offset, size_t remove, PureSpan insert) { - uint64_t size = draft->root.bytes; - if (insert.size && !insert.data) return PURE_INVALID; - if (offset > size || remove > size - offset || insert.size > PURE_OBJECT_LIMIT - (size - remove)) - return PURE_LIMIT; - if (!remove && !insert.size) return PURE_OK; - Entry next; - bool ok = rebuild(&draft->tree, draft->root, offset, remove, insert, &next); - if (ok) draft->root = next; - collect(draft); - return ok ? PURE_OK : PURE_STORE_ERROR; -} - -PureVersion pure_draft_version(const PureDraft *draft) { - return (PureVersion){draft->root.ref, draft->root.bytes}; -} - -PureResult pure_draft_read_version(const PureDraft *draft, PureVersion version, size_t offset, - size_t length, bool (*visit)(void *, PureSpan), void *context) { - Entry root = {version.ref, version.bytes}; - unsigned height; - if (offset > root.bytes || length > root.bytes - offset) return PURE_LIMIT; - if (!length) return PURE_OK; - if (!root_height(&draft->tree, root, &height)) return PURE_STORE_ERROR; - return visit_range(&draft->tree, root, height, offset, length, visit, context) ? - PURE_OK : PURE_STORE_ERROR; -} - -PureResult pure_draft_read(const PureDraft *draft, size_t offset, size_t length, - bool (*visit)(void *, PureSpan), void *context) { - return pure_draft_read_version(draft, pure_draft_version(draft), offset, length, visit, context); -} - -void pure_draft_restore(PureDraft *draft, PureVersion version) { - draft->root = (Entry){version.ref, version.bytes}; -} - -void pure_draft_hold(PureDraft *draft, bool held) { - draft->held = held; - if (!held) collect(draft); -} - -/* ---- Group commits ------------------------------------------------------ - * Threads that commit at the same time share one commit: they queue, the - * first becomes the leader and publishes every queued request of the same - * store with one head (one pair of syncs); the others wait for its result. - * Each request stays all or nothing on its own: a request with a stale draft - * is left out, and so is one whose key an earlier request of the same group - * already changes. The leader touches the waiters' drafts and handles only - * while they wait. */ -typedef struct Waiter { - PureDraft *const *drafts; - size_t count; - int64_t *revisions; - PureResult result; - bool accepted, done; - struct Waiter *next; -} Waiter; - -static struct { - pthread_mutex_t lock; - pthread_cond_t finished, joined; - Waiter *head, *tail; - size_t queued, expected; /* expected: how many requests groups have held lately */ - bool leading; - unsigned long groups, requests; -} group = {PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, PTHREAD_COND_INITIALIZER, NULL, NULL, 0, 1, false, 0, 0}; - -void pure_store_group_counts(unsigned long *groups, unsigned long *requests) { - pthread_mutex_lock(&group.lock); - *groups = group.groups; - *requests = group.requests; - pthread_mutex_unlock(&group.lock); -} - -enum { GATHER_NS = 2000000 }; - -/* Publish the queued requests of the store at path in one session. */ -static void commit_group(const char *path, Waiter *first) { - size_t total = 0; - for (Waiter *w = first; w; w = w->next) - if (!strcmp(w->drafts[0]->path, path)) { total += w->count; w->accepted = false; } - Session session; - Writer *writer = NULL; - Entry *roots = malloc(total * sizeof(*roots)); - const char **keys = malloc(total * sizeof(*keys)); - int64_t *revisions = malloc(total * sizeof(*revisions)); - PureDraft **drafts = malloc(total * sizeof(*drafts)); - bool allocated = roots && keys && revisions && drafts, checked = false; - PureResult result = allocated ? open_session(path, &session) : PURE_ALLOCATION; - size_t count = 0; - if (result == PURE_OK) { - /* A request holds when each of its keys is at its draft's base - * revision and no earlier request of this group changes it. */ - for (Waiter *w = first; w; w = w->next) { - if (strcmp(w->drafts[0]->path, path)) continue; - w->result = PURE_OK; - for (size_t i = 0; i < w->count && w->result == PURE_OK; ++i) { - PureDraft *d = w->drafts[i]; - Entry current; - uint64_t index; - int64_t stored; - bool found; - if (!find(&session.tree, (Entry){session.head.catalog, session.head.catalog_bytes}, d->key, &index, - &found, ¤t, &stored)) { w->result = PURE_STORE_ERROR; break; } - for (size_t k = 0; k < count; ++k) - if (!strcmp(keys[k], d->key)) { ++stored; break; } /* it moves with the earlier request */ - if (stored != d->revision) { w->revisions[i] = stored; w->result = PURE_CONFLICT; } - else if (stored == INT64_MAX) w->result = PURE_LIMIT; - } - if (w->result != PURE_OK) continue; - w->accepted = true; - for (size_t i = 0; i < w->count; ++i) { - drafts[count] = w->drafts[i]; - roots[count] = w->drafts[i]->root; - keys[count] = w->drafts[i]->key; - revisions[count++] = w->drafts[i]->revision; - } - } - checked = true; - if (count) { - result = PURE_ALLOCATION; - if ((writer = malloc(sizeof(*writer)))) { - *writer = (Writer){.fd = session.fd, .at = commit_start(&session)}; - bool written = true; - for (size_t i = 0; i < count && written; ++i) { - /* After compaction a draft's stored references name the old file. */ - bool relocate = session.file.st_dev != drafts[i]->device || session.file.st_ino != drafts[i]->inode; - written = write_root(&drafts[i]->tree, writer, &roots[i], relocate); - } - result = written ? publish(&session, writer, count, keys, roots, revisions) : PURE_STORE_ERROR; - } - } - /* Rebase while still holding the writer lock; the arenas are no longer - * needed. Drafts of a store handle share one new mapping. If remapping - * fails a draft keeps equal content at its new revision. */ - PureStore *refreshed[8] = {NULL}; - size_t fresh = 0; - for (size_t i = 0; result == PURE_OK && i < count; ++i) { - PureStore *store = drafts[i]->store; - bool rebased; - if (store) { - bool seen = false; - for (size_t k = 0; k < fresh && !seen; ++k) seen = refreshed[k] == store; - if (!seen && refresh(store) == PURE_OK) { - seen = true; - if (fresh < 8) refreshed[fresh++] = store; - } - rebased = seen && rebase(drafts[i]) == PURE_OK; - } else rebased = load_draft(drafts[i]) == PURE_OK; - if (!rebased) drafts[i]->revision = revisions[i]; - } - } - /* Before the checks every request shares the failure; after them an - * accepted request takes the publication's result and new revisions, and - * a refused one keeps its own result. */ - size_t at = 0; - for (Waiter *w = first; w; w = w->next) { - if (strcmp(w->drafts[0]->path, path)) continue; - if (!checked) { w->result = result; continue; } - if (!w->accepted) continue; - for (size_t i = 0; i < w->count; ++i) w->revisions[i] = revisions[at++]; - w->result = result; - } - free(writer); - free(roots); - free((void *)keys); - free(revisions); - free(drafts); - if (allocated) close_session(&session); -} - -PureResult pure_draft_commit_all(PureDraft *const *drafts, size_t count, int64_t *revisions) { - for (size_t i = 0; i < count; ++i) { - revisions[i] = drafts[i]->revision; - if (!drafts[i]->path || strcmp(drafts[i]->path, drafts[0]->path)) return PURE_INVALID; - for (size_t k = 0; k < i; ++k) if (!strcmp(drafts[k]->key, drafts[i]->key)) return PURE_INVALID; - } - if (!count) return PURE_OK; - Waiter me = {.drafts = drafts, .count = count, .revisions = revisions}; - pthread_mutex_lock(&group.lock); - if (group.tail) group.tail->next = &me; else group.head = &me; - group.tail = &me; - ++group.queued; - pthread_cond_signal(&group.joined); - while (!me.done) { - if (group.leading) { pthread_cond_wait(&group.finished, &group.lock); continue; } - group.leading = true; - /* When several threads have been committing together, give the others - * a moment (at most 2 ms) to join, or they would miss this group and - * wait for the next one: a sync costs far more. One thread never waits. */ - if (group.expected > 1 && group.queued < group.expected) { - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_nsec += GATHER_NS; - if (deadline.tv_nsec >= 1000000000) { ++deadline.tv_sec; deadline.tv_nsec -= 1000000000; } - while (group.queued < group.expected && - pthread_cond_timedwait(&group.joined, &group.lock, &deadline) == 0) {} - } - Waiter *batch = group.head; - size_t size = group.queued; - group.head = group.tail = NULL; - group.queued = 0; - ++group.groups; - group.requests += size; - /* Remember the largest recent group, forgetting one member per group. */ - group.expected = size > group.expected - 1 ? size : group.expected - 1; - if (!group.expected) group.expected = 1; - pthread_mutex_unlock(&group.lock); - /* One session per store among the queued requests. */ - for (Waiter *w = batch; w; w = w->next) { - bool earlier = false; - for (Waiter *v = batch; v != w && !earlier; v = v->next) earlier = !strcmp(v->drafts[0]->path, w->drafts[0]->path); - if (!earlier) commit_group(w->drafts[0]->path, batch); - } - pthread_mutex_lock(&group.lock); - for (Waiter *w = batch, *next; w; w = next) { next = w->next; w->done = true; } - group.leading = false; - pthread_cond_broadcast(&group.finished); - } - pthread_mutex_unlock(&group.lock); - return me.result; -} - -PureResult pure_draft_commit(PureDraft *draft, int64_t *revision) { - return pure_draft_commit_all(&draft, 1, revision); -} - -PureResult pure_store_put(const char *path, const char *key, int64_t expected, - PureSpan body, int64_t *revision) { - return mutate(path, key, expected, body, false, 0, 0, PURE_OBJECT_LIMIT, revision); -} - -PureResult pure_store_splice(const char *path, const char *key, int64_t expected, - size_t offset, size_t remove, PureSpan insert, - size_t max_size, int64_t *revision) { - return mutate(path, key, expected, insert, true, offset, remove, max_size, revision); -} - -#ifdef PURE_STORE_TEST -static bool shape(const Tree *tree, Entry entry, unsigned height, bool root, - uint64_t *leaves, uint64_t *small) { - if (!height) { - ++*leaves; - if (!root && entry.bytes < LEAF_MIN) ++*small; - return leaf_bytes(tree, entry) != NULL; - } - Entry kids[FAN_MAX]; - size_t count = children(tree, entry, height, kids); - if (!count || (root && count < 2)) return false; - if (!root && count < FAN_MIN) ++*small; - for (size_t i = 0; i < count; ++i) - if (!shape(tree, kids[i], height - 1, false, leaves, small)) return false; - return true; -} - -/* Test-only: live arena nodes and their payload bytes. */ -void pure_draft_test_arena(const PureDraft *draft, size_t *nodes, size_t *bytes) { - *nodes = *bytes = 0; - for (size_t i = 0; i < draft->tree.count; ++i) { - const Node *node = &draft->tree.nodes[i]; - if (node->height == FREE) continue; - ++*nodes; - *bytes += node->bytes ? node->size : node->size * sizeof(Entry); - } -} - -/* Test-only: the stored height, leaf count and number of underfull nodes. */ -bool pure_tree_test_shape(const char *path, const char *key, unsigned *height, - uint64_t *leaves, uint64_t *small, uint64_t *file_end) { - int directory = open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); - int fd = directory >= 0 ? openat(directory, tree_name, O_RDONLY | O_CLOEXEC) : -1; - if (directory >= 0) close(directory); - Tree tree = {0}; - Head head; - struct stat file; - Entry document; - uint64_t index; - int64_t revision; - bool found = false; - *leaves = *small = 0; - bool ok = fd >= 0 && read_head(fd, &file, false, &head, NULL) && map_tree(fd, &file, &head, &tree) && - find(&tree, (Entry){head.catalog, head.catalog_bytes}, key, &index, &found, - &document, &revision) && found && root_height(&tree, document, height) && - (!document.bytes || shape(&tree, document, *height, true, leaves, small)); - if (ok) *file_end = head.end; - release_tree(&tree); - if (fd >= 0) close(fd); - return ok; -} -#endif diff --git a/prototypes/connections/store.h b/prototypes/connections/store.h deleted file mode 100644 index dbfa8f8..0000000 --- a/prototypes/connections/store.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef PURE_STORE_H -#define PURE_STORE_H -#include "flow.h" -#include "result.h" -#include - -#define PURE_OBJECT_LIMIT ((2u * 6u + 32u) * 32768u + 8192u) -typedef struct { - void *mapping; - size_t mapped_size, object_size; - PureSpan body; - int64_t revision; -} PureStoredView; - -PureResult pure_store_init(const char *path); -PureResult pure_store_get(const char *path, const char *key, size_t available, - PureStoredView *out); -PureResult pure_store_get_range(const char *path, const char *key, size_t offset, - size_t length, size_t available, PureStoredView *out); -/* Inspect once: guards and mapping refer to the same immutable file version. - * minimum preserves bounds checks from eliminated intermediate views. - * whole selects the suffix starting at offset; otherwise length is exact. */ -PureResult pure_store_select(const char *path, const char *key, size_t offset, - size_t length, bool whole, size_t minimum, - size_t available, PureStoredView *out); -void pure_store_release(PureStoredView *); -PureResult pure_store_put(const char *path, const char *key, int64_t expected, - PureSpan body, int64_t *revision); -/* Atomic byte replacement: retain [0,offset), insert bytes, retain suffix after - * remove bytes. Check the expected revision and bounds under the same writer - * lock. expected=0 permits creating an empty base (offset=remove=0). - * Retained bytes use bounded scratch space or a selected platform file copy; - * never assembles a whole new object in application memory. The immutable - * file is still rewritten. max_size is the final-body policy from the - * prepared app and is checked with the current size under the writer lock. */ -PureResult pure_store_splice(const char *path, const char *key, int64_t expected, - size_t offset, size_t remove, PureSpan insert, - size_t max_size, int64_t *revision); - -/* Visit each stored key once; the order is store-defined. Returning false - * from visit stops early. key is valid only during the call. Hosts that - * browse documents use this instead of reading a store's file layout. */ -PureResult pure_store_keys(const char *path, bool (*visit)(void *, const char *key), - void *context); - -/* Fault hooks exist only in the separately built unit-test binary. */ -typedef enum { - PURE_AFTER_HEADER, PURE_BEFORE_FILE_SYNC, PURE_BEFORE_RENAME, PURE_BEFORE_DIR_SYNC, - PURE_COPY_RANGE_UNSUPPORTED_AFTER_PARTIAL -} PureStoreStage; -#ifdef PURE_STORE_TEST -int pure_store_test_fault(PureStoreStage stage); -#endif -#endif diff --git a/prototypes/connections/test-module.c b/prototypes/connections/test-module.c deleted file mode 100644 index ccb3371..0000000 --- a/prototypes/connections/test-module.c +++ /dev/null @@ -1,2853 +0,0 @@ -#define _POSIX_C_SOURCE 200809L -#include "module.h" -#include "module-store.h" -#include "module-text.h" -#include "store.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#define CHECK(x) do { if (!(x)) { fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #x); abort(); } } while (0) - -/* Store faults: a stage fails (or the process dies there when crash is set). */ -static int fail_stage = -1; -static int crash; -int pure_store_test_fault(PureStoreStage stage) { - if ((int)stage != fail_stage) return 0; - if (crash) _exit(77); - return 1; -} - -typedef struct { const char *name; double initial; } S; -typedef struct { const char *name; uint32_t type; } F; -/* kind 0 is a key input; otherwise PURE_KIND_MESSAGE with fields. */ -typedef struct { const char *name, *key; int kind; bool keyed; const F *fields; size_t nfields; } I; -#define KEY(name, key) {name, key, 0, false, NULL, 0} -/* A timer's period (milliseconds) rides in nfields. */ -#define TIMER(name, period) {name, NULL, PURE_KIND_TIMER, false, NULL, period} -typedef struct { const char *name; uint32_t start, length; } V; -typedef struct { uint32_t event, start, length; } H; -typedef struct { uint8_t op; uint32_t arg; double value; } O; -/* Text (type 1, ref = constant), cursor (type 2, ref = text state), list - * (type 3, ref = record) and index (type 4, ref = list state) states follow - * the number states; text views follow the number views. */ -typedef struct { const char *name; uint32_t type, ref; } T; /* type: the state type, its flags from bit 8 */ -/* A record field: a number and its default, or text (text = 1) whose - * default is the constant numbered initial. */ -typedef struct { const char *name; uint32_t text; double initial; } RF; -typedef struct { const char *name; const RF *fields; size_t nfields; } R; -/* A further view: a table (type 2, start = list state) or a column of the - * view numbered table - 1. */ -typedef struct { const char *name; uint32_t type, table, start, length; } W; -typedef struct { - const S *s; size_t ns; const I *in; size_t ni; const V *v; size_t nv; - const H *h; size_t nh; const O *o; size_t no; const char *const *c; size_t nc; - const T *t; size_t nt; const char *const *k; size_t nk; const V *tv; size_t ntv; - const R *r; size_t nr; const W *w; size_t nw; -} Spec; - -static unsigned char *put32(unsigned char *p, uint32_t v) { - for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(v >> (8 * i)); - return p + 4; -} - -static unsigned char *put_double(unsigned char *p, double value) { - uint64_t bits; - memcpy(&bits, &value, sizeof(bits)); - put32(p, (uint32_t)bits); - return put32(p + 4, (uint32_t)(bits >> 32)); -} - -/* Test-side encoder for format v5; the toolchain has its own. */ -static size_t build(unsigned char *out, Spec spec) { - unsigned char strings[4096], blob[4096]; - size_t used = 0, blob_used = 0; - const char *text[512]; - uint32_t offset[512]; - size_t names = 0; - for (size_t i = 0; i < spec.ns; ++i) text[names++] = spec.s[i].name; - for (size_t i = 0; i < spec.nt; ++i) text[names++] = spec.t[i].name; - for (size_t i = 0; i < spec.nc; ++i) text[names++] = spec.c[i]; - for (size_t i = 0; i < spec.ni; ++i) { - text[names++] = spec.in[i].name; - if (!spec.in[i].kind) text[names++] = spec.in[i].key; - for (size_t f = 0; spec.in[i].kind != PURE_KIND_TIMER && f < spec.in[i].nfields; ++f) - text[names++] = spec.in[i].fields[f].name; - } - for (size_t i = 0; i < spec.nv; ++i) text[names++] = spec.v[i].name; - for (size_t i = 0; i < spec.ntv; ++i) text[names++] = spec.tv[i].name; - for (size_t i = 0; i < spec.nw; ++i) text[names++] = spec.w[i].name; - size_t record_names = names, record_fields = 0; - for (size_t i = 0; i < spec.nr; ++i) { - text[names++] = spec.r[i].name; - for (size_t f = 0; f < spec.r[i].nfields; ++f) text[names++] = spec.r[i].fields[f].name; - record_fields += spec.r[i].nfields; - } - for (size_t i = 0; i < names; ++i) { - size_t length = strlen(text[i]); - offset[i] = (uint32_t)used; - strings[used++] = (unsigned char)length; - memcpy(strings + used, text[i], length); - used += length; - } - size_t fields = 0; - for (size_t i = 0; i < spec.ni; ++i) fields += spec.in[i].kind == PURE_KIND_TIMER ? 0 : spec.in[i].nfields; - for (size_t i = 0; i < spec.nk; ++i) { memcpy(blob + blob_used, spec.k[i], strlen(spec.k[i])); blob_used += strlen(spec.k[i]); } - unsigned char *p = out; - memset(p, 0, 40); - memcpy(p, "PUREMOD5", 8); - size_t states = spec.ns + spec.nt, views = spec.nv + spec.ntv + spec.nw; - p[8] = (unsigned char)states; p[9] = (unsigned char)(states >> 8); - p[10] = (unsigned char)spec.ni; p[12] = (unsigned char)views; p[14] = (unsigned char)spec.nh; - put32(p + 16, (uint32_t)spec.no); - put32(p + 20, (uint32_t)used); - p[24] = (unsigned char)fields; p[26] = (unsigned char)spec.nc; p[28] = (unsigned char)spec.nk; - p[30] = (unsigned char)spec.nr; p[36] = (unsigned char)record_fields; - put32(p + 32, (uint32_t)blob_used); - p += 40; - size_t n = 0; - for (size_t i = 0; i < spec.ns; ++i) { p = put32(p, offset[n++]); p = put32(p, 0); p = put_double(p, spec.s[i].initial); } - for (size_t i = 0; i < spec.nt; ++i) { - p = put32(p, offset[n++]); - p = put32(p, spec.t[i].type & 0xFF); - p = put32(p, spec.t[i].ref); - p = put32(p, spec.t[i].type >> 8); - } - for (size_t i = 0; i < spec.nc; ++i) { p = put32(p, offset[n++]); p = put32(p, 1); p = put32(p, 1); } - uint32_t field_offsets[64], field_start = 0; - size_t nf = 0; - for (size_t i = 0; i < spec.ni; ++i) { - const I *in = &spec.in[i]; - p = put32(p, offset[n++]); - *p++ = (unsigned char)(in->kind ? in->kind : 1); *p++ = in->keyed; *p++ = 0; *p++ = 0; - bool timer = in->kind == PURE_KIND_TIMER; - p = put32(p, timer ? (uint32_t)in->nfields : in->kind ? 0 : offset[n++]); - uint32_t first = in->kind == PURE_KIND_MESSAGE || in->kind == PURE_KIND_OUTPUT ? field_start : 0; /* only messages and outputs have fields */ - *p++ = (unsigned char)first; *p++ = (unsigned char)(first >> 8); - *p++ = (unsigned char)(timer ? 0 : in->nfields); *p++ = 0; - for (size_t f = 0; !timer && f < in->nfields; ++f) field_offsets[nf++] = offset[n++]; - field_start += timer ? 0 : (uint32_t)in->nfields; - } - nf = 0; - for (size_t i = 0; i < spec.ni; ++i) - for (size_t f = 0; spec.in[i].kind != PURE_KIND_TIMER && f < spec.in[i].nfields; ++f) { - p = put32(p, field_offsets[nf++]); - p = put32(p, spec.in[i].fields[f].type); - } - for (size_t i = 0, at = 0; i < spec.nk; ++i) { p = put32(p, (uint32_t)at); p = put32(p, (uint32_t)strlen(spec.k[i])); at += strlen(spec.k[i]); } - size_t record_field = record_names; - for (size_t i = 0, first = 0; i < spec.nr; ++i) { - p = put32(p, offset[record_field]); - record_field += 1 + spec.r[i].nfields; - *p++ = (unsigned char)first; *p++ = 0; *p++ = (unsigned char)spec.r[i].nfields; *p++ = 0; - first += spec.r[i].nfields; - } - for (size_t i = 0, n = record_names; i < spec.nr; ++i, ++n) - for (size_t f = 0; f < spec.r[i].nfields; ++f) { - const RF *field = &spec.r[i].fields[f]; - p = put32(p, offset[++n]); - p = put32(p, field->text); - if (field->text) { p = put32(p, (uint32_t)field->initial); p = put32(p, 0); } - else p = put_double(p, field->initial); - } - for (size_t i = 0; i < spec.nv; ++i) { p = put32(p, offset[n++]); p = put32(p, 0); p = put32(p, spec.v[i].start); p = put32(p, spec.v[i].length); } - for (size_t i = 0; i < spec.ntv; ++i) { p = put32(p, offset[n++]); p = put32(p, 1); p = put32(p, spec.tv[i].start); p = put32(p, spec.tv[i].length); } - for (size_t i = 0; i < spec.nw; ++i) { - const W *w = &spec.w[i]; - p = put32(p, offset[n++]); - *p++ = (unsigned char)w->type; *p++ = (unsigned char)(w->type >> 8); /* a filtered table: its list + 1 */ - *p++ = (unsigned char)w->table; *p++ = (unsigned char)(w->table >> 8); - p = put32(p, w->start); - p = put32(p, w->length); - } - for (size_t i = 0; i < spec.nh; ++i) { p = put32(p, spec.h[i].event); p = put32(p, spec.h[i].start); p = put32(p, spec.h[i].length); } - for (size_t i = 0; i < spec.no; ++i) { - *p++ = spec.o[i].op; *p++ = 0; *p++ = 0; *p++ = 0; - p = put32(p, spec.o[i].arg); - p = put_double(p, spec.o[i].value); - } - memcpy(p, strings, used); - memcpy(p + used, blob, blob_used); - return (size_t)(p + used + blob_used - out); -} - -static PureModule module_a, module_b; -static char error[128]; - -static PureResult load(Spec spec, PureModule *out) { - static unsigned char bytes[65536]; - size_t size = build(bytes, spec); - return pure_module_load(bytes, size, out, error, sizeof(error)); -} - -#define LEN(a) (sizeof(a) / sizeof(*(a))) -#define SPEC(s, in, v, h, o) ((Spec){s, LEN(s), in, LEN(in), v, LEN(v), h, LEN(h), o, LEN(o), NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}) - -enum { C = PURE_OP_CONST, L = PURE_OP_LOAD, DT = PURE_OP_DT, ADD = PURE_OP_ADD, - MUL = PURE_OP_MUL, DIV = PURE_OP_DIV, GT = PURE_OP_GT, MIN = PURE_OP_MIN, - MAX = PURE_OP_MAX, CL = PURE_OP_CLAMP, ST = PURE_OP_STORE, JZ = PURE_OP_JZ, - JMP = PURE_OP_JMP, NEG = PURE_OP_NEG }; - -static const S counter_states[] = {{"count", 0}, {"x", 3}}; -static const I counter_inputs[] = {KEY("tap", "Space"), KEY("move", "ArrowRight")}; -static const V counter_views[] = {{"label", 0, 1}, {"width", 1, 6}}; -static const O counter_code[] = { - {L, 0, 0}, /* 0 view label */ - {C, 0, 40}, {L, 0, 0}, {C, 0, 20}, {MIN, 0, 0}, {C, 0, 8}, /* 1 view width */ - {MUL, 0, 0}, - /* 7: fixed-up below: 40 + min(count, 20) * 8 needs an ADD; see width2 */ - {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, /* 7 tap.down */ - /* 11 move.down: if x > 5 { x = 0 } else { x = x + 1 } */ - {L, 1, 0}, {C, 0, 5}, {GT, 0, 0}, {JZ, 18, 0}, {C, 0, 0}, {ST, 1, 0}, {JMP, 22, 0}, - {L, 1, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 1, 0}, - /* 22 frame: x = x + dt */ - {L, 1, 0}, {DT, 0, 0}, {ADD, 0, 0}, {ST, 1, 0}, - /* 26 view width2 = 40 + min(count, 20) * 8 */ - {C, 0, 40}, {L, 0, 0}, {C, 0, 20}, {MIN, 0, 0}, {C, 0, 8}, {MUL, 0, 0}, {ADD, 0, 0}, -}; -static const H counter_handlers[] = {{1, 7, 4}, {3, 11, 11}, {0, 22, 4}}; - -static void behaviour(void) { - /* width at 1..7 leaves two values: rejected at load. */ - CHECK(load(SPEC(counter_states, counter_inputs, counter_views, counter_handlers, counter_code), &module_a) == PURE_INVALID); - CHECK(strstr(error, "view must leave one value of its type")); - static const V views[] = {{"label", 0, 1}, {"width", 26, 7}}; - CHECK(load(SPEC(counter_states, counter_inputs, views, counter_handlers, counter_code), &module_a) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - double value; - CHECK(pure_app_view(&app, 1, &value) && value == 40); - for (int i = 0; i < 25; ++i) CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); - CHECK(app.values[0] == 25 && pure_app_view(&app, 1, &value) && value == 200); - CHECK(pure_app_input(&app, 0, PURE_EDGE_UP) == PURE_OK && app.events == 26); - /* if / else with forward JZ and JMP */ - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 4); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 5); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 6); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 0); - CHECK(pure_app_frame(&app) == PURE_OK && app.values[1] == 1.0 / 60.0 && app.ticks == 1); - CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_INVALID); - CHECK(pure_module_input(&module_a, "move") == 1 && pure_module_view(&module_a, "width") == 1 && - pure_module_state(&module_a, "missing") == -1); - - /* Replacement keeps values by name. */ - static const S next_states[] = {{"fresh", 7}, {"count", 99}}; - static const I no_inputs[1]; - static const V no_views[1]; - static const H no_handlers[1]; - static const O no_code[1]; - CHECK(load((Spec){next_states, 2, no_inputs, 0, no_views, 0, no_handlers, 0, no_code, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, &module_b) == PURE_OK); - CHECK(pure_app_replace(&app, &module_b) == PURE_OK); - CHECK(app.module == &module_b && app.values[0] == 7 && app.values[1] == 25); - CHECK(pure_app_replace(&app, &module_a) == PURE_OK); - CHECK(app.values[0] == 25 && app.values[1] == 3); - pure_app_stop(&app); -} - -/* A fault leaves every state exactly as before the event. */ -static void faults(void) { - static const S states[] = {{"a", 1}, {"b", 2}}; - static const I inputs[] = {KEY("one", "KeyA"), KEY("two", "KeyB"), KEY("three", "Digit3")}; - static const O code[] = { - {C, 0, 9}, {ST, 0, 0}, {C, 0, 1}, {C, 0, 0}, {DIV, 0, 0}, {ST, 1, 0}, /* 0 divide */ - {C, 0, 9}, {ST, 0, 0}, {C, 0, 1e308}, {C, 0, 10}, {MUL, 0, 0}, {ST, 1, 0}, /* 6 overflow */ - {C, 0, 9}, {ST, 0, 0}, {C, 0, 1}, {C, 0, 3}, {C, 0, 2}, {CL, 0, 0}, {ST, 1, 0}, /* 12 clamp */ - {C, 0, 5}, {ST, 0, 0}, /* 19 ok */ - {C, 0, 0}, {NEG, 0, 0}, {C, 0, 0}, {MIN, 0, 0}, /* 21 view -0 */ - {C, 0, 0}, {C, 0, 0}, {NEG, 0, 0}, {MAX, 0, 0}, /* 25 view +0 */ - }; - static const H handlers[] = {{1, 0, 6}, {3, 6, 6}, {5, 12, 7}, {5, 19, 2}}; - static const V views[] = {{"low", 21, 4}, {"high", 25, 4}}; - CHECK(load(SPEC(states, inputs, views, handlers, code), &module_a) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - const PureFault expected[] = {PURE_FAULT_DIVIDE, PURE_FAULT_NON_FINITE, PURE_FAULT_CLAMP}; - for (uint32_t i = 0; i < 3; ++i) { - CHECK(pure_app_input(&app, i, PURE_EDGE_DOWN) == PURE_INVALID && app.fault == expected[i]); - CHECK(app.values[0] == 1 && app.values[1] == 2 && app.events == 0); - } - /* A later handler of the same event does not run after a fault. */ - CHECK(app.fault_pc == 17); - double value; - CHECK(pure_app_view(&app, 0, &value) && value == 0 && signbit(value)); - CHECK(pure_app_view(&app, 1, &value) && value == 0 && !signbit(value)); - pure_app_stop(&app); -} - -typedef struct { const char *reason; Spec spec; PureResult result; } Rejection; - -static void rejections(void) { - static const S s1[] = {{"a", 0}}; - static const S dup[] = {{"a", 0}, {"a", 1}}; - static const S bad_name[] = {{"1a", 0}}; - static const S long_name[] = {{"abcdefghijklmnopqrstuvwxyz012345", 0}}; - static const S non_finite[] = {{"a", INFINITY}}; - static const I i1[] = {KEY("k", "Space")}; - static const I bad_key[] = {KEY("k", "KeyAA")}; - static const I dup_key[] = {KEY("k", "Space"), KEY("j", "Space")}; - static const V clash[] = {{"a", 0, 1}}; - static const V v_store[] = {{"v", 0, 2}}; - static const V v_dt[] = {{"v", 2, 1}}; - static const V v_empty[] = {{"v", 0, 0}}; - static const H none[1]; - static const O load_a[] = {{L, 0, 0}}; - static const O store[] = {{C, 0, 1}, {ST, 0, 0}, {DT, 0, 0}}; - static const H h0[] = {{0, 0, 2}}; - static const H h_bad_event[] = {{3, 0, 2}}; - static const O back[] = {{C, 0, 1}, {JZ, 0, 0}}; - static const O beyond[] = {{C, 0, 1}, {JZ, 3, 0}}; - static const O under[] = {{ADD, 0, 0}}; - static const O join[] = {{C, 0, 1}, {JZ, 4, 0}, {C, 0, 1}, {C, 0, 1}, {ST, 0, 0}}; - static const O leftover[] = {{C, 0, 1}, {C, 0, 2}}; - static const O unknown_state[] = {{L, 1, 0}, {ST, 0, 0}}; - static const O nan_const[] = {{C, 0, NAN}, {ST, 0, 0}}; - static const O noncanon[] = {{C, 0, 1}, {NEG, 1, 0}, {ST, 0, 0}}; - static const O unreachable[] = {{JMP, 2, 0}, {C, 0, 1}, {C, 0, 1}, {ST, 0, 0}}; - static const O bad_op[] = {{99, 0, 0}, {ST, 0, 0}}; - static const H h_all[] = {{0, 0, 1}}; - static const H h_two[] = {{0, 0, 2}}; - static const H h_five[] = {{0, 0, 5}}; - static const H h_four[] = {{0, 0, 4}}; - O overflow[66]; - for (int i = 0; i < 65; ++i) overflow[i] = (O){C, 0, 1}; - overflow[65] = (O){ST, 0, 0}; - const Rejection cases[] = { - {"duplicate name", {dup, 2, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"state name", {bad_name, 1, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"state name", {long_name, 1, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"non-finite initial value", {non_finite, 1, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"input key", {s1, 1, bad_key, 1, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"duplicate key", {s1, 1, dup_key, 2, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"duplicate name", {s1, 1, i1, 0, clash, 1, none, 0, load_a, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"view with an effect", {s1, 1, i1, 0, v_store, 1, none, 0, store, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"view with an effect", {s1, 1, i1, 0, v_dt, 1, none, 0, store, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"empty view", {s1, 1, i1, 0, v_empty, 1, none, 0, store, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"handler event", {s1, 1, i1, 1, clash, 0, h_bad_event, 1, store, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"branch not forward", {s1, 1, i1, 0, clash, 0, h_two, 1, back, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"branch not forward", {s1, 1, i1, 0, clash, 0, h_two, 1, beyond, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"stack underflow", {s1, 1, i1, 0, clash, 0, h_all, 1, under, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"join with different stacks", {s1, 1, i1, 0, clash, 0, h_five, 1, join, 5, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"handler must leave no value", {s1, 1, i1, 0, clash, 0, h_two, 1, leftover, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"no such state", {s1, 1, i1, 0, clash, 0, h_two, 1, unknown_state, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"non-canonical operand", {s1, 1, i1, 0, clash, 0, h_two, 1, nan_const, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"non-canonical operand", {s1, 1, i1, 0, clash, 0, h0, 1, noncanon, 3, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"unreachable instruction", {s1, 1, i1, 0, clash, 0, h_four, 1, unreachable, 4, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"unknown opcode", {s1, 1, i1, 0, clash, 0, h_two, 1, bad_op, 2, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - {"stack overflow", {s1, 1, i1, 0, clash, 0, (H[]){{0, 0, 66}}, 1, overflow, 66, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, PURE_INVALID}, - }; - for (size_t i = 0; i < LEN(cases); ++i) { - PureResult result = load(cases[i].spec, &module_a); - if (result != cases[i].result || !strstr(error, cases[i].reason)) { - fprintf(stderr, "case %zu: expected '%s', got %d '%s'\n", i, cases[i].reason, (int)result, error); - abort(); - } - } - static S many[33]; - static char many_names[33][8]; - for (unsigned i = 0; i < 33; ++i) { snprintf(many_names[i], 8, "s%u", i % 100u); many[i] = (S){many_names[i], 0}; } - CHECK(load((Spec){many, 33, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}, &module_a) == PURE_LIMIT); - static unsigned char bytes[65536]; - size_t size = build(bytes, (Spec){s1, 1, i1, 0, clash, 0, none, 0, load_a, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}); - CHECK(pure_module_load(bytes, size - 1, &module_a, error, sizeof(error)) == PURE_INVALID); - CHECK(pure_module_load(bytes, size + 1, &module_a, error, sizeof(error)) == PURE_INVALID); - bytes[0] = 'X'; - CHECK(pure_module_load(bytes, size, &module_a, error, sizeof(error)) == PURE_INVALID); -} - -static uint32_t next_random(uint32_t *state) { - *state ^= *state << 13; - *state ^= *state >> 17; - *state ^= *state << 5; - return *state; -} - -/* Arbitrary damage is either rejected or yields a module that runs safely. */ -static void mutations(void) { - static const V views[] = {{"label", 0, 1}, {"width", 26, 7}}; - static unsigned char valid[65536], bytes[65536]; - size_t size = build(valid, SPEC(counter_states, counter_inputs, views, counter_handlers, counter_code)); - uint32_t state = 0x2545f491u; - unsigned accepted = 0; - for (unsigned round = 0; round < 20000; ++round) { - memcpy(bytes, valid, size); - unsigned flips = 1 + next_random(&state) % 4; - for (unsigned f = 0; f < flips; ++f) - bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); - if (pure_module_load(bytes, size, &module_a, error, sizeof(error)) != PURE_OK) continue; - ++accepted; - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - for (unsigned e = 0; e < 16; ++e) { - if (e % 3 == 0) pure_app_frame(&app); - else pure_app_input(&app, e % 4, e & 1 ? PURE_EDGE_UP : PURE_EDGE_DOWN); - for (uint32_t v = 0; v < module_a.views; ++v) { double value; pure_app_view(&app, v, &value); } - for (uint32_t s = 0; s < module_a.states; ++s) CHECK(isfinite(app.values[s])); - } - pure_app_stop(&app); - } - printf("{\"module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); -} - -static Spec editor_spec; - -/* An in-memory data host: every splice makes a new immutable version. */ -typedef struct { char key[65]; unsigned char *bytes; size_t size; int64_t revision; } Stored; -typedef struct { - Stored docs[8]; - size_t count, limit; - bool busy; - unsigned opened, closed; -} Memory; -typedef struct { - Memory *memory; - int index; /* -1 for an absent key */ - int64_t base; - unsigned char *versions[64]; - size_t sizes[64], count; -} Doc; - -static Stored *stored(Memory *m, const char *key) { - for (size_t i = 0; i < m->count; ++i) if (!strcmp(m->docs[i].key, key)) return &m->docs[i]; - return NULL; -} - -static PureResult fake_open(void *context, uint32_t collection, const char *key, void **out, int64_t *revision) { - Memory *m = context; - CHECK(collection == 0); - Doc *d = calloc(1, sizeof(*d)); - Stored *s = stored(m, key); - d->memory = m; - d->index = s ? (int)(s - m->docs) : -1; - if (!s) { - CHECK(m->count < 8); - d->index = (int)m->count; - snprintf(m->docs[m->count++].key, 65, "%s", key); - s = &m->docs[d->index]; - } - d->base = s->revision; - d->versions[0] = malloc(s->size + 1); - memcpy(d->versions[0], s->bytes ? s->bytes : (unsigned char *)"", s->size); - d->sizes[0] = s->size; - d->count = 1; - ++m->opened; - *out = d; - *revision = s->revision; - return PURE_OK; -} - -static PureVersion fake_version(void *context, void *document) { - (void)context; - Doc *d = document; - return (PureVersion){d->count - 1, d->sizes[d->count - 1]}; -} - -/* Two spans per read, so replies arrive in pieces. */ -static PureResult fake_read(void *context, void *document, PureVersion at, size_t offset, size_t length, - bool (*visit)(void *, PureSpan), void *visit_context) { - (void)context; - Doc *d = document; - CHECK(at.ref < d->count && at.bytes == d->sizes[at.ref] && offset + length <= at.bytes); - size_t half = length / 2; - if (half && !visit(visit_context, (PureSpan){d->versions[at.ref] + offset, half})) return PURE_OK; - if (length - half) visit(visit_context, (PureSpan){d->versions[at.ref] + offset + half, length - half}); - return PURE_OK; -} - -static PureResult fake_splice(void *context, void *document, size_t offset, size_t remove, PureSpan insert) { - Memory *m = context; - Doc *d = document; - size_t size = d->sizes[d->count - 1], next = size - remove + insert.size; - if (next > m->limit) return PURE_LIMIT; - CHECK(d->count < 64); - unsigned char *bytes = malloc(next + 1); - memcpy(bytes, d->versions[d->count - 1], offset); - if (insert.size) memcpy(bytes + offset, insert.data, insert.size); - memcpy(bytes + offset + insert.size, d->versions[d->count - 1] + offset + remove, size - offset - remove); - d->versions[d->count] = bytes; - d->sizes[d->count++] = next; - return PURE_OK; -} - -static PureResult fake_commit(void *context, void *document, int64_t *revision) { - Memory *m = context; - Doc *d = document; - Stored *s = &m->docs[d->index]; - *revision = s->revision; - if (m->busy) return PURE_BUSY; - if (s->revision != d->base) return PURE_CONFLICT; - free(s->bytes); - s->size = d->sizes[d->count - 1]; - s->bytes = malloc(s->size + 1); - memcpy(s->bytes, d->versions[d->count - 1], s->size); - *revision = ++s->revision; - /* Like a tree draft, a commit rebases: earlier versions are gone. */ - for (size_t i = 0; i + 1 < d->count; ++i) { free(d->versions[i]); d->versions[i] = NULL; d->sizes[i] = 0; } - d->versions[0] = d->versions[d->count - 1]; - d->sizes[0] = d->sizes[d->count - 1]; - d->count = 1; - d->base = s->revision; - return PURE_OK; -} - -static void fake_close(void *context, void *document) { - Memory *m = context; - Doc *d = document; - for (size_t i = 0; i < d->count; ++i) free(d->versions[i]); - free(d); - ++m->closed; -} - -static PureData fake_data(Memory *m) { - return (PureData){m, fake_open, fake_version, fake_read, fake_splice, fake_commit, fake_close, NULL, NULL}; -} - -typedef struct { unsigned char bytes[256]; size_t size, spans; } Collected; - -static bool count_bytes(void *context, PureSpan span) { - *(size_t *)context += span.size; - return true; -} - -static bool collect(void *context, PureSpan span) { - Collected *c = context; - CHECK(c->size + span.size <= sizeof(c->bytes)); - memcpy(c->bytes + c->size, span.data, span.size); - c->size += span.size; - ++c->spans; - return true; -} - -static bool replied(const PureApp *app, const char *text) { - Collected c = {{0}, 0, 0}; - CHECK(pure_app_reply(app, collect, &c) == PURE_OK); - return c.size == strlen(text) && !memcmp(c.bytes, text, c.size); -} - -static PureSpan edit_payload(unsigned char *buffer, uint32_t start, uint32_t remove, const char *insert) { - put32(buffer, start); - put32(buffer + 4, remove); - memcpy(buffer + 8, insert, strlen(insert)); - return (PureSpan){buffer, 8 + strlen(insert)}; -} - -enum { FIELD = PURE_OP_FIELD, DOC = PURE_OP_DOC, SIZE = PURE_OP_DOC_SIZE, LEN_ = PURE_OP_LEN, - REPL = PURE_OP_REPLACE, REPLY = PURE_OP_REPLY, REQ = PURE_OP_REQUIRE, LE = PURE_OP_LE }; - -/* The document editor as a module: show replies the document, edit splices - * start .. start + remove with insert, replies insert and counts edits. A - * keep message replies the document read before its own edit; guarded edits - * require the final size to stay within 8 bytes. */ -static void messages(void) { - static const F edit_fields[] = {{"start", 1}, {"remove", 1}, {"insert", 2}}; - static const I inputs[] = { - {"show", NULL, PURE_KIND_MESSAGE, true, NULL, 0}, - {"edit", NULL, PURE_KIND_MESSAGE, true, edit_fields, 3}, - {"keep", NULL, PURE_KIND_MESSAGE, true, edit_fields + 2, 1}, - {"guarded", NULL, PURE_KIND_MESSAGE, true, edit_fields, 3}, - KEY("tick", "Space"), - }; - static const S states[] = {{"edits", 0}}; - static const char *const collections[] = {"documents"}; - static const O code[] = { - {DOC, 0, 0}, {REPLY, 0, 0}, /* 0 show */ - {FIELD, 0, 0}, {FIELD, 0, 0}, {FIELD, 1, 0}, {ADD, 0, 0}, {FIELD, 2, 0}, /* 2 edit */ - {REPL, 0, 0}, {FIELD, 2, 0}, {REPLY, 0, 0}, {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, - {DOC, 0, 0}, {REPLY, 0, 0}, {C, 0, 0}, {C, 0, 0}, {FIELD, 0, 0}, {REPL, 0, 0}, /* 14 keep: prepend */ - {FIELD, 0, 0}, {FIELD, 0, 0}, {FIELD, 1, 0}, {ADD, 0, 0}, {FIELD, 2, 0}, /* 20 guarded */ - {REPL, 0, 0}, {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, - {SIZE, 0, 0}, {C, 0, 8}, {LE, 0, 0}, {REQ, 0, 0}, - }; - static const H handlers[] = {{1, 0, 2}, {3, 2, 12}, {5, 14, 6}, {7, 20, 14}}; - static const V views[] = {{"count", 0, 0}}; - Spec spec = {states, 1, inputs, 5, views, 0, handlers, 4, code, LEN(code), collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; - editor_spec = spec; - PureResult loaded = load(spec, &module_a); - if (loaded != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } - Memory memory = {.limit = 100}; - PureApp app; - CHECK(pure_app_start(&app, &module_a, fake_data(&memory)) == PURE_OK); - unsigned char buffer[64]; - - CHECK(pure_app_message(&app, 0, "doc", -1, (PureSpan){0}) == PURE_NOT_FOUND && app.revision == 0); - CHECK(pure_app_message(&app, 1, "doc", 0, edit_payload(buffer, 0, 0, "hello")) == PURE_OK); - CHECK(app.revision == 1 && replied(&app, "hello") && app.values[0] == 1); - CHECK(pure_app_message(&app, 0, "doc", -1, (PureSpan){0}) == PURE_OK && app.revision == 1 && replied(&app, "hello")); - /* stale base conflicts before bounds; bounds fail as limit; nothing changes */ - CHECK(pure_app_message(&app, 1, "doc", 0, edit_payload(buffer, 99, 0, "x")) == PURE_CONFLICT && app.revision == 1); - CHECK(pure_app_message(&app, 1, "doc", 1, edit_payload(buffer, 99, 0, "x")) == PURE_LIMIT && app.revision == 1); - CHECK(pure_app_message(&app, 1, "doc", 1, edit_payload(buffer, 2, 9, "x")) == PURE_LIMIT); - CHECK(pure_app_message(&app, 1, "doc", 1, (PureSpan){buffer, 7}) == PURE_LIMIT && app.revision == 0); - CHECK(pure_app_message(&app, 1, "doc", -1, edit_payload(buffer, 0, 0, "x")) == PURE_INVALID); - CHECK(pure_app_message(&app, 1, "bad/key", 1, edit_payload(buffer, 0, 0, "x")) == PURE_INVALID); - CHECK(pure_app_message(&app, 4, "doc", 1, (PureSpan){0}) == PURE_INVALID); - CHECK(memory.docs[0].revision == 1 && memory.docs[0].size == 5 && app.values[0] == 1); - /* A busy commit changes neither the document nor the states. */ - memory.busy = true; - CHECK(pure_app_message(&app, 1, "doc", 1, edit_payload(buffer, 5, 0, "!")) == PURE_BUSY); - memory.busy = false; - CHECK(memory.docs[0].size == 5 && app.values[0] == 1 && app.events == 2); - /* The size limit of the data host fails as limit. */ - memory.limit = 6; - CHECK(pure_app_message(&app, 1, "doc", 1, edit_payload(buffer, 5, 0, "!!")) == PURE_LIMIT); - memory.limit = 100; - /* A reply read before the event's own edit keeps its content after commit. */ - CHECK(pure_app_message(&app, 2, "doc", 1, (PureSpan){(const unsigned char *)">> ", 3}) == PURE_OK); - CHECK(app.revision == 2 && replied(&app, "hello")); - CHECK(pure_app_message(&app, 0, "doc", -1, (PureSpan){0}) == PURE_OK && replied(&app, ">> hello")); - /* require: an edit that would exceed 8 bytes changes nothing */ - CHECK(pure_app_message(&app, 3, "doc", 2, edit_payload(buffer, 8, 0, "!")) == PURE_LIMIT); - CHECK(memory.docs[0].size == 8 && app.values[0] == 1); - CHECK(pure_app_message(&app, 3, "doc", 2, edit_payload(buffer, 0, 3, "")) == PURE_OK && app.revision == 3); - CHECK(app.values[0] == 2); - /* Another key is its own document. */ - CHECK(pure_app_message(&app, 1, "other", 0, edit_payload(buffer, 0, 0, "abc")) == PURE_OK && app.revision == 1); - pure_app_stop(&app); - CHECK(memory.opened == memory.closed); - for (size_t i = 0; i < memory.count; ++i) free(memory.docs[i].bytes); - - /* Verifier rules for typed operations. */ - static const O field_in_key[] = {{FIELD, 0, 0}, {ST, 0, 0}}; - static const H key_handler[] = {{9, 0, 2}}; - Spec bad = spec; - bad.o = field_in_key; bad.no = 2; bad.h = key_handler; bad.nh = 1; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "message operation outside")); - static const O num_insert[] = {{C, 0, 0}, {C, 0, 0}, {C, 0, 1}, {REPL, 0, 0}}; - static const H edit_handler[] = {{3, 0, 4}}; - bad.o = num_insert; bad.no = 4; bad.h = edit_handler; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); - static const O bytes_state[] = {{FIELD, 2, 0}, {ST, 0, 0}}; - static const H edit_handler2[] = {{3, 0, 2}}; - bad.o = bytes_state; bad.no = 2; bad.h = edit_handler2; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); - static const O far_field[] = {{FIELD, 3, 0}, {REPLY, 0, 0}}; - bad.o = far_field; bad.h = edit_handler2; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "no such state, field")); - static const I unkeyed[] = {{"note", NULL, PURE_KIND_MESSAGE, false, NULL, 0}}; - static const O doc_read[] = {{DOC, 0, 0}, {REPLY, 0, 0}}; - static const H note_handler[] = {{1, 0, 2}}; - bad = (Spec){states, 1, unkeyed, 1, views, 0, note_handler, 1, doc_read, 2, collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "document operation outside")); - static const F unknown_type[] = {{"insert", 5}, {"start", 1}}; - static const I misplaced[] = {{"edit", NULL, PURE_KIND_MESSAGE, true, unknown_type, 2}}; - bad = (Spec){states, 1, misplaced, 1, views, 0, NULL, 0, NULL, 0, collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "message field")); - static const H even[] = {{2, 0, 0}}; - bad = (Spec){states, 1, inputs, 1, views, 0, even, 1, NULL, 0, collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "handler event")); - static const V field_view[] = {{"v", 0, 1}}; - static const O field_only[] = {{FIELD, 0, 0}}; - bad = (Spec){states, 1, inputs, 2, field_view, 1, NULL, 0, field_only, 1, collections, 1, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view with an effect")); - /* A value may cross a join when every path brings the same stack. */ - static const O branch_value[] = {{C, 0, 1}, {C, 0, 1}, {JZ, 3, 0}, {ST, 0, 0}}; - static const O branch_mismatch[] = {{C, 0, 1}, {C, 0, 1}, {JZ, 4, 0}, {C, 0, 2}, {ST, 0, 0}}; - static const H frame_handler[] = {{0, 0, 4}}; - bad = (Spec){states, 1, inputs, 0, views, 0, frame_handler, 1, branch_value, 4, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_OK); - static const H frame_five[] = {{0, 0, 5}}; - bad.o = branch_mismatch; bad.no = 5; bad.h = frame_five; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "join with different stacks")); -} - -/* Damaged message modules either fail to load or run messages safely. */ -static void message_mutations(void) { - static unsigned char valid[65536], bytes[65536]; - size_t size = build(valid, editor_spec); - uint32_t state = 0x9e3779b9u; - unsigned accepted = 0; - for (unsigned round = 0; round < 20000; ++round) { - memcpy(bytes, valid, size); - unsigned flips = 1 + next_random(&state) % 3; - for (unsigned f = 0; f < flips; ++f) - bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); - if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; - ++accepted; - Memory memory = {.limit = 64}; - PureApp app; - CHECK(pure_app_start(&app, &module_b, fake_data(&memory)) == PURE_OK); - unsigned char payload[32]; - for (unsigned e = 0; e < 12; ++e) { - for (size_t k = 0; k < sizeof(payload); ++k) payload[k] = (unsigned char)next_random(&state); - put32(payload, next_random(&state) % 12); - put32(payload + 4, next_random(&state) % 4); - PureSpan span = {payload, next_random(&state) % sizeof(payload)}; - uint32_t input = next_random(&state) % 5; - if (module_b.input[input].kind == PURE_KIND_MESSAGE) { - int64_t base = (int64_t)(next_random(&state) % 4) - 1; - if (pure_app_message(&app, input, "doc", base, span) == PURE_OK) { - size_t reply = 0; - CHECK(pure_app_reply(&app, count_bytes, &reply) == PURE_OK && reply <= 256); - } - } else pure_app_input(&app, input, e & 1 ? PURE_EDGE_UP : PURE_EDGE_DOWN); - pure_app_frame(&app); - for (uint32_t s = 0; s < module_b.states; ++s) CHECK(isfinite(app.values[s])); - } - pure_app_stop(&app); - CHECK(memory.opened == memory.closed); - for (size_t i = 0; i < memory.count; ++i) free(memory.docs[i].bytes); - } - printf("{\"message_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); -} - -static Spec text_spec; - -typedef struct { char text[256]; size_t size; } Text; - -static bool text_span(void *context, PureSpan span) { - Text *t = context; - CHECK(t->size + span.size < sizeof(t->text)); - memcpy(t->text + t->size, span.data, span.size); - t->size += span.size; - t->text[t->size] = 0; - return true; -} - -static const char *view_text(PureApp *app, uint32_t view) { - static Text t; - t.size = 0; - t.text[0] = 0; - CHECK(pure_app_view_text(app, view, text_span, &t) == PURE_OK); - return t.text; -} - -static const char *state_text(const PureApp *app, uint32_t state) { - static Text t; - t.size = 0; - t.text[0] = 0; - CHECK(pure_draft_read(app->texts[state], 0, pure_draft_size(app->texts[state]), text_span, &t) == PURE_OK); - return t.text; -} - -static double number_view(PureApp *app, uint32_t view) { - double value; - CHECK(pure_app_view(app, view, &value)); - return value; -} - -enum { CUL = PURE_OP_CUR_LOAD, CS = PURE_OP_CUR_STORE, CM = PURE_OP_CUR_MOVE, TI = PURE_OP_TEXT_INPUT, - TINS = PURE_OP_TEXT_INSERT, TREM = PURE_OP_TEXT_REMOVE, TL = PURE_OP_TEXT_LOAD, TK = PURE_OP_TEXT_CONST, - CHR = PURE_OP_CHARS, NT = PURE_OP_NUM_TEXT, CAT = PURE_OP_CONCAT, TLN = PURE_OP_TEXT_LINE, - CC = PURE_OP_CUR_COUNT, CST = PURE_OP_CUR_START }; - -/* A small text editor module: states edits (num), note (text), at (cursor in - * note), title (text). The runtime keeps at on character boundaries and - * moves it with edits; a failed event restores text and cursor. */ -static void texts(void) { - static const S nums[] = {{"edits", 0}}; - static const T textual[] = {{"note", 1, 0}, {"at", 2, 1}, {"title", 1, 1}}; - static const char *const constants[] = {"\xea\xb0\x80" "a\nbc", "\xeb\xa9\x94\xeb\xaa\xa8", "XYZ", - " \xc2\xb7 ", "\xec\x9e\x90"}; - static const I inputs[] = { - {"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("left", "ArrowLeft"), KEY("right", "ArrowRight"), - KEY("up", "ArrowUp"), KEY("down", "ArrowDown"), KEY("back", "Backspace"), KEY("home", "Home"), KEY("bad", "KeyB"), - }; - static const O code[] = { - {CUL, 2, 0}, {TI, 0, 0}, {TINS, 1, 0}, {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, /* 0 typed */ - {CUL, 2, 0}, {C, 0, -1}, {CM, 0, 0}, {CS, 2, 0}, /* 7 left */ - {CUL, 2, 0}, {C, 0, 1}, {CM, 0, 0}, {CS, 2, 0}, /* 11 right */ - {CUL, 2, 0}, {C, 0, -1}, {CM, 1, 0}, {CS, 2, 0}, /* 15 up */ - {CUL, 2, 0}, {C, 0, 1}, {CM, 1, 0}, {CS, 2, 0}, /* 19 down */ - {CUL, 2, 0}, {C, 0, -1}, {CM, 0, 0}, {CUL, 2, 0}, {TREM, 1, 0}, /* 23 back */ - {CST, 1, 0}, {CS, 2, 0}, /* 28 home */ - {CUL, 2, 0}, {TK, 2, 0}, {TINS, 1, 0}, {C, 0, 1}, {C, 0, 0}, {DIV, 0, 0}, {ST, 0, 0}, /* 30 bad */ - {TL, 1, 0}, {CHR, 0, 0}, /* 37 view chars */ - {CUL, 2, 0}, {CC, 0, 0}, /* 39 view offset */ - {CUL, 2, 0}, {CC, 1, 0}, /* 41 view line_no */ - {CUL, 2, 0}, {TLN, 1, 0}, /* 43 view line */ - {TL, 3, 0}, {TK, 3, 0}, {CAT, 0, 0}, {TL, 1, 0}, {CHR, 0, 0}, {NT, 0, 0}, {CAT, 0, 0}, - {TK, 4, 0}, {CAT, 0, 0}, /* 45 view header */ - {C, 0, 1}, {C, 0, 3}, {DIV, 0, 0}, {NT, 0, 0}, /* 54 view ratio */ - }; - static const H handlers[] = {{1, 0, 7}, {3, 7, 4}, {5, 11, 4}, {7, 15, 4}, {9, 19, 4}, {11, 23, 5}, - {13, 28, 2}, {15, 30, 7}}; - static const V number_views[] = {{"chars", 37, 2}, {"offset", 39, 2}, {"line_no", 41, 2}}; - static const V text_views[] = {{"line", 43, 2}, {"header", 45, 9}, {"ratio", 54, 4}}; - Spec spec = {nums, 1, inputs, LEN(inputs), number_views, 3, handlers, LEN(handlers), code, LEN(code), - NULL, 0, textual, 3, constants, LEN(constants), text_views, 3, NULL, 0, NULL, 0}; - text_spec = spec; - if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - CHECK(!strcmp(state_text(&app, 1), "\xea\xb0\x80" "a\nbc") && app.values[2] == 0); - for (int i = 0; i < 6; ++i) CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK); - CHECK(app.values[2] == 7); /* stops at the end */ - CHECK(number_view(&app, 0) == 5 && number_view(&app, 2) == 2 && !strcmp(view_text(&app, 3), "bc")); - CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 4); /* up keeps column 2 */ - const char *han = "\xed\x95\x9c"; - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)han, 3}) == PURE_OK); - CHECK(!strcmp(state_text(&app, 1), "\xea\xb0\x80" "a\xed\x95\x9c\nbc") && app.values[2] == 7 && app.values[0] == 1); - CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 4); /* back over 3 bytes */ - CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 3); - CHECK(!strcmp(state_text(&app, 1), "\xea\xb0\x80\nbc")); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); /* stops at the start */ - CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); /* nothing before */ - CHECK(pure_app_input(&app, 4, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 4); /* down to column 0 */ - /* A failing event restores the text, the cursor and the states. */ - CHECK(pure_app_input(&app, 7, PURE_EDGE_DOWN) == PURE_INVALID && app.fault == PURE_FAULT_DIVIDE); - CHECK(!strcmp(state_text(&app, 1), "\xea\xb0\x80\nbc") && app.values[2] == 4 && app.values[0] == 1); - CHECK(!strcmp(view_text(&app, 4), "\xeb\xa9\x94\xeb\xaa\xa8 \xc2\xb7 4\xec\x9e\x90")); - CHECK(!strcmp(view_text(&app, 5), "0.3333333333333333")); - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xff", 1}) == PURE_INVALID); - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xed\xa0\x80", 3}) == PURE_INVALID); /* surrogate */ - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xf0\x9f\x98\x80", 4}) == PURE_OK); - CHECK(app.values[2] == 8 && number_view(&app, 1) == 3); /* offset in characters */ - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 4); /* back over the emoji */ - CHECK(pure_app_input(&app, 6, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); - /* Replacement keeps the text and its cursor by name. */ - static const S fresh[] = {{"fresh", 7}}; - static const T kept[] = {{"note", 1, 0}, {"at", 2, 1}}; - static const char *const other[] = {"ignored"}; - Spec next = {fresh, 1, inputs, 0, number_views, 0, handlers, 0, code, 0, NULL, 0, kept, 2, other, 1, text_views, 0, NULL, 0, NULL, 0}; - CHECK(load(next, &module_b) == PURE_OK); - CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 3); - CHECK(pure_app_replace(&app, &module_b) == PURE_OK); - CHECK(app.values[0] == 7 && app.values[2] == 3 && !strcmp(state_text(&app, 1), "\xea\xb0\x80\n\xf0\x9f\x98\x80" "bc")); - pure_app_stop(&app); - - /* Verification of text operations. */ - static const T two[] = {{"one", 1, 0}, {"two", 1, 0}, {"c", 2, 1}}; - static const O wrong_text[] = {{CUL, 3, 0}, {TK, 0, 0}, {TINS, 2, 0}}; - static const H frame[] = {{0, 0, 3}}; - Spec bad = {nums, 1, inputs, 0, number_views, 0, frame, 1, wrong_text, 3, NULL, 0, two, 3, constants, 1, text_views, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "cursor of another text")); - static const O input_in_frame[] = {{CUL, 3, 0}, {TI, 0, 0}, {TINS, 1, 0}}; - bad.o = input_in_frame; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "text input outside")); - static const O store_text[] = {{TK, 0, 0}, {ST, 0, 0}}; - static const H frame2[] = {{0, 0, 2}}; - bad.o = store_text; bad.no = 2; bad.h = frame2; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); - static const O load_text[] = {{L, 1, 0}, {ST, 0, 0}}; - bad.o = load_text; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "no such state")); - static const V text_as_number[] = {{"v", 0, 1}}; - static const O constant_only[] = {{TK, 0, 0}}; - bad = (Spec){nums, 1, inputs, 0, text_as_number, 1, NULL, 0, constant_only, 1, NULL, 0, two, 3, constants, 1, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view must leave one value of its type")); - static const T cursor_on_number[] = {{"c", 2, 0}}; - bad = (Spec){nums, 1, inputs, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, cursor_on_number, 1, constants, 1, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); - static const char *const broken[] = {"\xc0\xaf"}; - bad = (Spec){nums, 1, inputs, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, broken, 1, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "text constant")); -} - -static bool ignore_span(void *context, PureSpan span) { - (void)context; - CHECK(pure_utf8_valid(span) || span.size); - return true; -} - -/* Damaged text modules either fail to load or edit text safely: every cursor - * stays within its text and every text stays valid UTF-8. */ -static void text_mutations(void) { - static unsigned char valid[65536], bytes[65536]; - size_t size = build(valid, text_spec); - static const char *const words[] = {"a", "\xea\xb0\x80", "\n", "\xf0\x9f\x98\x80", "xy\nz", ""}; - uint32_t state = 0x1b873593u; - unsigned accepted = 0; - for (unsigned round = 0; round < 20000; ++round) { - memcpy(bytes, valid, size); - unsigned flips = 1 + next_random(&state) % 3; - for (unsigned f = 0; f < flips; ++f) - bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); - if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; - ++accepted; - PureApp app; - if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; - for (unsigned e = 0; e < 16; ++e) { - uint32_t input = next_random(&state) % 8; - if (module_b.input[input].kind == PURE_KIND_TEXT) { - const char *word = words[next_random(&state) % LEN(words)]; - pure_app_text(&app, input, (PureSpan){(const unsigned char *)word, strlen(word)}); - } else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); - else pure_app_frame(&app); - for (uint32_t v = 0; v < module_b.views; ++v) { - double value; - if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); - else pure_app_view(&app, v, &value); - } - for (uint32_t s = 0; s < module_b.states; ++s) { - if (module_b.state_type[s] == PURE_STATE_CURSOR) { - PureDraft *text = app.texts[module_b.state_ref[s]]; - size_t length = pure_draft_size(text), at = (size_t)app.values[s]; - CHECK(app.values[s] >= 0 && at <= length && app.values[s] == (double)at); - if (at < length) { /* never inside a multi-byte character */ - Text t = {{0}, 0}; - pure_draft_read(text, at, 1, text_span, &t); - CHECK(((unsigned char)t.text[0] & 0xC0) != 0x80); - } - } else if (module_b.state_type[s] == PURE_STATE_TEXT) { - unsigned utf8 = 0; - bool ok = true; - Text t = {{0}, 0}; - if (pure_draft_size(app.texts[s]) < 200) { - pure_draft_read(app.texts[s], 0, pure_draft_size(app.texts[s]), text_span, &t); - ok = pure_utf8_step(&utf8, (PureSpan){(const unsigned char *)t.text, t.size}) && - pure_utf8_complete(utf8); - } - CHECK(ok); - } else CHECK(isfinite(app.values[s])); - } - } - pure_app_stop(&app); - } - printf("{\"text_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); -} - -static Spec list_spec; - -enum { LC = PURE_OP_LIST_COUNT, LA = PURE_OP_LIST_APPEND, LI = PURE_OP_LIST_INSERT, LR = PURE_OP_LIST_REMOVE, - IG = PURE_OP_ITEM_GET, IS = PURE_OP_ITEM_SET, XL = PURE_OP_INDEX_LOAD, XS = PURE_OP_INDEX_STORE, - XM = PURE_OP_INDEX_MOVE, XF = PURE_OP_INDEX_FIRST, XE = PURE_OP_INDEX_LAST, XN = PURE_OP_INDEX_NUM, - NOT = PURE_OP_NOT, TITLE = 1, DONE = 1 | 1u << 16 }; - -static const char *item_title(const PureApp *app, uint32_t list, size_t at) { - static char title[64]; - PureCell cell = pure_list_get(app->lists[list], at, 0); - CHECK(cell.size < sizeof(title)); - if (cell.size) memcpy(title, cell.text, cell.size); - title[cell.size] = 0; - return title; -} - -/* A task list: made (num), tasks (list of task {title, done}), at (index of - * tasks). Indices follow their item across inserts and removes; a failed - * event undoes every list change. */ -static void lists(void) { - static const S nums[] = {{"made", 0}}; - static const T states[] = {{"tasks", 3, 0}, {"at", 4, 1}}; - static const RF task_fields[] = {{"title", 1, 0}, {"done", 0, 0}}; - static const R records[] = {{"task", task_fields, 2}}; - static const char *const constants[] = {"", "new", "x"}; - static const I inputs[] = { - {"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("ins", "KeyI"), KEY("del", "Delete"), - KEY("up", "ArrowUp"), KEY("down", "ArrowDown"), KEY("toggle", "Space"), KEY("rename", "KeyR"), KEY("bad", "KeyB"), - }; - static const O code[] = { - {TI, 0, 0}, {C, 0, 0}, {LA, 1, 0}, {XE, 1, 0}, {XS, 2, 0}, {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, /* 0 typed */ - {XL, 2, 0}, {TK, 1, 0}, {C, 0, 0}, {LI, 1, 0}, /* 9 ins */ - {XL, 2, 0}, {LR, 1, 0}, /* 13 del */ - {XL, 2, 0}, {C, 0, -1}, {XM, 0, 0}, {XS, 2, 0}, /* 15 up */ - {XL, 2, 0}, {C, 0, 1}, {XM, 0, 0}, {XS, 2, 0}, /* 19 down */ - {XL, 2, 0}, {XL, 2, 0}, {IG, DONE, 0}, {NOT, 0, 0}, {IS, DONE, 0}, /* 23 toggle */ - {XL, 2, 0}, {XL, 2, 0}, {IG, TITLE, 0}, {TK, 2, 0}, {CAT, 0, 0}, {IS, TITLE, 0}, /* 28 rename */ - {XL, 2, 0}, {TK, 2, 0}, {IS, TITLE, 0}, {XL, 2, 0}, {LR, 1, 0}, {TK, 1, 0}, {C, 0, 0}, /* 34 bad */ - {LA, 1, 0}, {C, 0, 1}, {C, 0, 0}, {DIV, 0, 0}, {ST, 0, 0}, - {LC, 1, 0}, /* 46 view count */ - {XL, 2, 0}, {XN, 0, 0}, /* 47 view position */ - {XL, 2, 0}, {IG, DONE, 0}, /* 49 view done */ - {XL, 2, 0}, {IG, TITLE, 0}, /* 51 view current */ - }; - static const H handlers[] = {{1, 0, 9}, {3, 9, 4}, {5, 13, 2}, {7, 15, 4}, {9, 19, 4}, {11, 23, 5}, - {13, 28, 6}, {15, 34, 12}}; - static const V number_views[] = {{"count", 46, 1}, {"position", 47, 2}, {"done", 49, 2}}; - static const V text_views[] = {{"current", 51, 2}}; - Spec spec = {nums, 1, inputs, LEN(inputs), number_views, 3, handlers, LEN(handlers), code, LEN(code), - NULL, 0, states, 2, constants, 3, text_views, 1, records, 1, NULL, 0}; - list_spec = spec; - if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - double value; - CHECK(number_view(&app, 0) == 0 && number_view(&app, 1) == 0 && !pure_app_view(&app, 2, &value)); - CHECK(pure_app_view_text(&app, 3, text_span, &(Text){{0}, 0}) == PURE_NOT_FOUND); /* no item yet */ - CHECK(app.fault == PURE_FAULT_NONE); /* views leave faults alone */ - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"a", 1}) == PURE_OK); - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"b", 1}) == PURE_OK); - CHECK(pure_list_count(app.lists[1]) == 2 && app.values[2] == 1 && app.values[0] == 2); - CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); - CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); /* stops at the first */ - /* Inserting before the current item: the index follows it. */ - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 1); - CHECK(!strcmp(item_title(&app, 1, 0), "new") && !strcmp(view_text(&app, 3), "a")); - CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && number_view(&app, 2) == 1); - CHECK(pure_app_input(&app, 6, PURE_EDGE_DOWN) == PURE_OK && !strcmp(view_text(&app, 3), "ax")); - CHECK(pure_app_input(&app, 4, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 2); - CHECK(pure_app_input(&app, 4, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 2); /* stops at the last */ - /* Removing the last item: the index moves to the new last. */ - CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 1 && pure_list_count(app.lists[1]) == 2); - CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0); - /* Removing the current item: the index names the next. */ - CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 0 && !strcmp(view_text(&app, 3), "ax")); - /* A failing event undoes a set, a remove and an append. */ - CHECK(pure_app_input(&app, 7, PURE_EDGE_DOWN) == PURE_INVALID && app.fault == PURE_FAULT_DIVIDE); - CHECK(pure_list_count(app.lists[1]) == 1 && !strcmp(item_title(&app, 1, 0), "ax") && - pure_list_get(app.lists[1], 0, 1).num == 1 && app.values[2] == 0 && app.values[0] == 2); - CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && pure_list_count(app.lists[1]) == 0 && app.values[2] == 0); - CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_NOT_FOUND && app.fault == PURE_FAULT_NOT_FOUND); - CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_NOT_FOUND); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && pure_list_count(app.lists[1]) == 1 && app.values[2] == 0); - for (int i = 0; i < 3; ++i) CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xea\xb0\x80", 3}) == PURE_OK); - CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 2); - CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK); - - /* Replacement keeps the list by name. The same record keeps the list - * itself; a changed record keeps each field by name and kind and gives new - * fields their default. */ - static const S no_nums[] = {{"made", 0}}; - Spec same = {no_nums, 1, inputs, 0, number_views, 0, handlers, 0, code, 0, NULL, 0, states, 2, constants, 3, - text_views, 0, records, 1, NULL, 0}; - CHECK(load(same, &module_b) == PURE_OK); - const PureList *before = app.lists[1]; - CHECK(pure_app_replace(&app, &module_b) == PURE_OK && app.lists[1] == before && app.values[2] == 2); - static const RF changed_fields[] = {{"done", 0, 0}, {"note", 1, 1}, {"title", 0, 7}, {"rank", 0, 5}}; - static const R changed[] = {{"task", changed_fields, 4}}; - Spec next = same; - next.r = changed; - CHECK(load(next, &module_a) == PURE_OK); - CHECK(pure_app_replace(&app, &module_a) == PURE_OK && pure_list_count(app.lists[1]) == 4 && app.values[2] == 2); - PureCell cell = pure_list_get(app.lists[1], 2, 0); - CHECK(cell.num == 1); /* done kept by name */ - cell = pure_list_get(app.lists[1], 2, 1); - CHECK(cell.size == 3 && !memcmp(cell.text, "new", 3)); /* note: its default */ - CHECK(pure_list_get(app.lists[1], 2, 2).num == 7); /* title is now a number: default */ - CHECK(pure_list_get(app.lists[1], 0, 3).num == 5); - /* A state whose kind changed starts again. */ - static const T swapped_kinds[] = {{"tasks", 1, 0}, {"at", 3, 0}}; - next.t = swapped_kinds; - CHECK(load(next, &module_b) == PURE_OK); - CHECK(pure_app_replace(&app, &module_b) == PURE_OK && !app.lists[1] && pure_draft_size(app.texts[1]) == 0 && - pure_list_count(app.lists[2]) == 0); - pure_app_stop(&app); - - /* Verification of list operations. */ - static const T two_lists[] = {{"a", 3, 0}, {"b", 3, 0}, {"ia", 4, 1}}; - static const O other_index[] = {{XL, 3, 0}, {LR, 2, 0}}; - static const H frame[] = {{0, 0, 2}}; - Spec bad = {nums, 1, inputs, 0, number_views, 0, frame, 1, other_index, 2, NULL, 0, two_lists, 3, constants, 3, - text_views, 0, records, 1, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "index of another list")); - static const O swapped[] = {{C, 0, 0}, {TK, 0, 0}, {LA, 1, 0}}; - static const H frame3[] = {{0, 0, 3}}; - bad.o = swapped; bad.no = 3; bad.h = frame3; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); - static const O short_record[] = {{TK, 0, 0}, {LA, 1, 0}}; - bad.o = short_record; bad.no = 2; bad.h = frame; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "stack underflow")); - static const O no_field[] = {{XL, 3, 0}, {IG, 1 | 2u << 16, 0}, {ST, 0, 0}}; - bad.o = no_field; bad.no = 3; bad.h = frame3; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "no such state")); - static const O index_as_number[] = {{XL, 3, 0}, {ST, 0, 0}}; - bad.o = index_as_number; bad.no = 2; bad.h = frame; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); - static const V append_view[] = {{"v", 0, 3}}; - bad = (Spec){nums, 1, inputs, 0, append_view, 1, NULL, 0, (O[]){{TK, 0, 0}, {C, 0, 0}, {LA, 1, 0}}, 3, NULL, 0, - two_lists, 3, constants, 3, NULL, 0, records, 1, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view with an effect")); - static const T index_of_number[] = {{"ia", 4, 0}}; - bad = (Spec){nums, 1, inputs, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, index_of_number, 1, constants, 3, NULL, 0, - records, 1, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); - static const T list_of_nothing[] = {{"a", 3, 1}}; - bad.t = list_of_nothing; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); - bad.t = NULL; bad.nt = 0; - static const RF repeated[] = {{"x", 0, 0}, {"x", 1, 0}}; - bad.r = (R[]){{"task", repeated, 2}}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "duplicate record field")); - static const RF far_default[] = {{"x", 1, 3}}; - bad.r = (R[]){{"task", far_default, 1}}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "record field default")); - static const RF infinite[] = {{"x", 0, INFINITY}}; - bad.r = (R[]){{"task", infinite, 1}}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "record field default")); - bad.r = (R[]){{"task", NULL, 0}}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "record")); - bad.r = (R[]){{"made", task_fields, 2}}; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "duplicate name")); -} - -/* Damaged list modules either fail to load or keep every index on an item - * of its list and every text field valid UTF-8. */ -static void list_mutations(void) { - static unsigned char valid[65536], bytes[65536]; - size_t size = build(valid, list_spec); - static const char *const words[] = {"a", "\xea\xb0\x80", "\n", ""}; - uint32_t state = 0x85ebca6bu; - unsigned accepted = 0; - for (unsigned round = 0; round < 20000; ++round) { - memcpy(bytes, valid, size); - unsigned flips = 1 + next_random(&state) % 3; - for (unsigned f = 0; f < flips; ++f) - bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); - if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; - ++accepted; - PureApp app; - if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; - for (unsigned e = 0; e < 24; ++e) { - uint32_t input = next_random(&state) % 8; - if (module_b.input[input].kind == PURE_KIND_TEXT) { - const char *word = words[next_random(&state) % LEN(words)]; - pure_app_text(&app, input, (PureSpan){(const unsigned char *)word, strlen(word)}); - } else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); - else pure_app_frame(&app); - for (uint32_t v = 0; v < module_b.views; ++v) { - double value; - if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); - else pure_app_view(&app, v, &value); - } - for (uint32_t s = 0; s < module_b.states; ++s) { - if (module_b.state_type[s] == PURE_STATE_INDEX) { - size_t count = pure_list_count(app.lists[module_b.state_ref[s]]); - CHECK(app.values[s] == floor(app.values[s]) && app.values[s] >= 0 && - app.values[s] <= (count ? (double)(count - 1) : 0)); - } else if (module_b.state_type[s] == PURE_STATE_LIST) { - const PureRecord *record = &module_b.record[module_b.state_ref[s]]; - for (size_t at = 0; at < pure_list_count(app.lists[s]); ++at) - for (unsigned f = 0; f < record->count; ++f) { - PureCell cell = pure_list_get(app.lists[s], at, f); - if (module_b.record_field_text[record->first + f]) - CHECK(pure_utf8_valid((PureSpan){cell.text, cell.size})); - else CHECK(isfinite(cell.num)); - } - } else if (module_b.state_type[s] == PURE_STATE_NUM) CHECK(isfinite(app.values[s])); - } - } - pure_app_stop(&app); - } - printf("{\"list_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); -} - -static Spec table_spec; - -static const char *cell_text(PureApp *app, uint32_t column, size_t row) { - static Text t; - t.size = 0; - t.text[0] = 0; - CHECK(pure_app_cell_text(app, column, row, text_span, &t) == PURE_OK); - return t.text; -} - -enum { ROW = PURE_OP_ROW, EQ = PURE_OP_EQ, LT = PURE_OP_LT, PICK = PURE_OP_PICK, SLIDE = PURE_OP_SLIDE }; - -/* A table view has a row per item; its columns read the row and may branch. - * A value crosses a join when every path brings the same stack. */ -static void tables(void) { - static const S nums[] = {{"n", 0}}; - static const T states[] = {{"tasks", 3, 0}, {"at", 4, 1}}; - static const RF task_fields[] = {{"title", 1, 0}, {"done", 0, 0}}; - static const R records[] = {{"task", task_fields, 2}}; - static const char *const constants[] = {"", "x", " ", "empty"}; - static const I inputs[] = {{"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("toggle", "Space")}; - static const O code[] = { - {TI, 0, 0}, {C, 0, 0}, {LA, 1, 0}, {XE, 1, 0}, {XS, 2, 0}, /* 0 typed */ - {XL, 2, 0}, {XL, 2, 0}, {IG, DONE, 0}, {NOT, 0, 0}, {IS, DONE, 0}, /* 5 toggle */ - {LC, 1, 0}, {C, 0, 0}, {GT, 0, 0}, {JZ, 17, 0}, {XL, 2, 0}, {IG, TITLE, 0}, {JMP, 18, 0}, - {TK, 3, 0}, /* 10 current */ - {ROW, 1, 0}, {IG, TITLE, 0}, /* 18 title */ - {ROW, 1, 0}, {IG, DONE, 0}, {JZ, 25, 0}, {TK, 1, 0}, {JMP, 26, 0}, {TK, 2, 0}, /* 20 mark */ - {ROW, 1, 0}, {XN, 0, 0}, /* 26 pos */ - {ROW, 1, 0}, {XN, 0, 0}, {XL, 2, 0}, {XN, 0, 0}, {EQ, 0, 0}, /* 28 here */ - }; - static const H handlers[] = {{1, 0, 5}, {3, 5, 5}}; - static const V text_views[] = {{"current", 10, 8}}; - static const W table[] = {{"board", 2, 0, 1, 0}, {"title", 1, 2, 18, 2}, {"mark", 1, 2, 20, 6}, - {"pos", 0, 2, 26, 2}, {"here", 0, 2, 28, 5}}; - Spec spec = {nums, 1, inputs, 2, NULL, 0, handlers, 2, code, LEN(code), NULL, 0, states, 2, constants, 4, - text_views, 1, records, 1, table, 5}; - table_spec = spec; - if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } - CHECK(pure_module_view(&module_a, "board") == 1 && pure_module_view(&module_a, "title") == -1 && - pure_module_column(&module_a, 1, "title") == 2 && pure_module_column(&module_a, 1, "current") == -1); - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - double value; - CHECK(pure_app_rows(&app, 1) == 0 && !strcmp(view_text(&app, 0), "empty")); /* the else branch */ - CHECK(pure_app_cell_text(&app, 2, 0, text_span, &(Text){{0}, 0}) == PURE_NOT_FOUND); - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"a", 1}) == PURE_OK); - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"\xeb\xb9\xb5", 3}) == PURE_OK); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); - CHECK(pure_app_rows(&app, 1) == 2 && !strcmp(view_text(&app, 0), "\xeb\xb9\xb5")); - CHECK(!strcmp(cell_text(&app, 2, 0), "a") && !strcmp(cell_text(&app, 2, 1), "\xeb\xb9\xb5")); - CHECK(!strcmp(cell_text(&app, 3, 0), " ") && !strcmp(cell_text(&app, 3, 1), "x")); - CHECK(pure_app_cell(&app, 4, 1, &value) && value == 1 && pure_app_cell(&app, 5, 0, &value) && value == 0 && - pure_app_cell(&app, 5, 1, &value) && value == 1); - CHECK(!pure_app_cell(&app, 4, 2, &value) && pure_app_cell_text(&app, 2, 2, text_span, &(Text){{0}, 0}) == PURE_NOT_FOUND); - /* Columns are read by row, other views without one. */ - CHECK(!pure_app_view(&app, 4, &value) && pure_app_view_text(&app, 2, text_span, &(Text){{0}, 0}) == PURE_INVALID); - CHECK(pure_app_cell_text(&app, 0, 0, text_span, &(Text){{0}, 0}) == PURE_INVALID && pure_app_rows(&app, 0) == 0); - CHECK(pure_app_view_text(&app, 1, text_span, &(Text){{0}, 0}) == PURE_INVALID); - pure_app_stop(&app); - - /* Verification of rows, tables and joins. */ - Spec bad = spec; - static const V row_view[] = {{"current", 18, 2}}; - bad.tv = row_view; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "row outside a column")); - static const H row_handler[] = {{3, 26, 2}}; - bad = spec; bad.h = row_handler; bad.nh = 1; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "row outside a column")); - static const W twice[] = {{"board", 2, 0, 1, 0}, {"title", 1, 2, 18, 2}, {"title", 1, 2, 18, 2}}; - bad = spec; bad.w = twice; bad.nw = 3; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "duplicate column")); - static const W two_tables[] = {{"board", 2, 0, 1, 0}, {"title", 1, 2, 18, 2}, {"other", 2, 0, 1, 0}, - {"title", 1, 4, 18, 2}, {"tasks", 0, 4, 26, 2}}; - bad = spec; bad.w = two_tables; bad.nw = 5; - CHECK(load(bad, &module_b) == PURE_OK); /* column names belong to their table */ - static const W not_a_table[] = {{"title", 1, 1, 18, 2}}; - bad = spec; bad.w = not_a_table; bad.nw = 1; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view type")); - static const W over_number[] = {{"board", 2, 0, 0, 0}}; - bad = spec; bad.w = over_number; bad.nw = 1; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "table list")); - static const O other_row[] = {{ROW, 2, 0}, {XN, 0, 0}}; - static const W other_row_column[] = {{"board", 2, 0, 1, 0}, {"pos", 0, 1, 0, 2}}; - bad = spec; bad.o = other_row; bad.no = 2; bad.h = NULL; bad.nh = 0; bad.tv = NULL; bad.ntv = 0; - bad.w = other_row_column; bad.nw = 2; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "no such state")); - static const O typed_join[] = {{L, 0, 0}, {JZ, 4, 0}, {TK, 0, 0}, {JMP, 5, 0}, {C, 0, 1}}; - static const V mixed[] = {{"current", 0, 5}}; - bad = spec; bad.o = typed_join; bad.no = 5; bad.h = NULL; bad.nh = 0; bad.tv = mixed; bad.w = NULL; bad.nw = 0; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "join with different stacks")); - /* At most 8 joins with a value may be ahead at once. */ - for (unsigned open = 8; open <= 9; ++open) { - static O nested[1 + 2 * 9 + 9 + 1]; - unsigned n = 0; - nested[n++] = (O){C, 0, 1}; - for (unsigned k = 0; k < open; ++k) { nested[n++] = (O){C, 0, 0}; nested[n++] = (O){JZ, 1 + 2 * open + k, 0}; } - for (unsigned k = 0; k < open; ++k) nested[n++] = (O){NEG, 0, 0}; - nested[n++] = (O){ST, 0, 0}; - H frame = {0, 0, n}; - bad = spec; bad.o = nested; bad.no = n; bad.h = &frame; bad.nh = 1; bad.tv = NULL; bad.ntv = 0; bad.w = NULL; bad.nw = 0; - CHECK(load(bad, &module_b) == (open == 8 ? PURE_OK : PURE_INVALID)); - } - CHECK(strstr(error, "more than 8 joins ahead")); - - /* PICK copies a value from below, SLIDE drops values under the top: a - * value used twice is computed once. (3 + 4) * (3 + 4) with a text between. */ - static const O picked[] = {{C, 0, 3}, {C, 0, 4}, {ADD, 0, 0}, {TK, 1, 0}, {PICK, 1, 0}, {PICK, 2, 0}, {MUL, 0, 0}, - {SLIDE, 2, 0}}; - static const V squared[] = {{"sq", 0, 8}}; - bad = (Spec){nums, 1, inputs, 0, squared, 1, NULL, 0, picked, 8, NULL, 0, NULL, 0, constants, 2, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(bad, &module_b) == PURE_OK); - CHECK(pure_app_start(&app, &module_b, (PureData){0}) == PURE_OK && number_view(&app, 0) == 49); - pure_app_stop(&app); - static const O pick_text[] = {{TK, 1, 0}, {C, 0, 1}, {PICK, 1, 0}, {SLIDE, 2, 0}}; - static const V text_left[] = {{"t", 0, 4}}; - bad.v = NULL; bad.nv = 0; bad.tv = text_left; bad.ntv = 1; bad.o = pick_text; bad.no = 4; - CHECK(load(bad, &module_b) == PURE_OK); - CHECK(pure_app_start(&app, &module_b, (PureData){0}) == PURE_OK && !strcmp(view_text(&app, 0), "x")); - pure_app_stop(&app); - static const O pick_type[] = {{TK, 1, 0}, {PICK, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {SLIDE, 1, 0}}; - static const V five[] = {{"sq", 0, 5}}, three[] = {{"sq", 0, 3}}; - bad.o = pick_type; bad.no = 5; bad.tv = NULL; bad.ntv = 0; bad.v = five; bad.nv = 1; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); /* a copy keeps its type */ - static const O pick_far[] = {{C, 0, 1}, {PICK, 1, 0}, {SLIDE, 1, 0}}; - bad.o = pick_far; bad.no = 3; bad.v = three; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "stack underflow")); - static const O slide_all[] = {{C, 0, 1}, {C, 0, 2}, {SLIDE, 2, 0}}; - bad.o = slide_all; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "stack underflow")); - static const O slide_none[] = {{C, 0, 1}, {C, 0, 2}, {SLIDE, 0, 0}}; - bad.o = slide_none; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "stack underflow")); -} - -/* Damaged table modules either fail to load or evaluate every cell safely. */ -static void table_mutations(void) { - static unsigned char valid[65536], bytes[65536]; - size_t size = build(valid, table_spec); - uint32_t state = 0xc2b2ae35u; - unsigned accepted = 0; - for (unsigned round = 0; round < 20000; ++round) { - memcpy(bytes, valid, size); - unsigned flips = 1 + next_random(&state) % 3; - for (unsigned f = 0; f < flips; ++f) - bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); - if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; - ++accepted; - PureApp app; - if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; - for (unsigned e = 0; e < 12; ++e) { - uint32_t input = next_random(&state) % 2; - if (module_b.input[input].kind == PURE_KIND_TEXT) pure_app_text(&app, input, (PureSpan){(const unsigned char *)"t", 1}); - else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); - for (uint32_t v = 0; v < module_b.views; ++v) { - double value; - if (!module_b.view_table[v]) { - if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); - else pure_app_view(&app, v, &value); - continue; - } - size_t rows = pure_app_rows(&app, module_b.view_table[v] - 1u); - for (size_t row = 0; row <= rows; ++row) { - if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_cell_text(&app, v, row, ignore_span, NULL); - else if (pure_app_cell(&app, v, row, &value)) CHECK(isfinite(value)); - } - } - } - pure_app_stop(&app); - } - printf("{\"table_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); -} - -/* ---- Stored states ---------------------------------------------------- */ - -static char *new_store(void) { - static char path[64]; - snprintf(path, sizeof(path), "/tmp/pure-module-store-XXXXXX"); - CHECK(mkdtemp(path) && pure_store_init(path) == PURE_OK); - return path; -} - -static void remove_store(const char *path) { - DIR *entries = opendir(path); - CHECK(entries); - struct dirent *entry; - while ((entry = readdir(entries))) - if (strcmp(entry->d_name, ".") && strcmp(entry->d_name, "..")) CHECK(unlinkat(dirfd(entries), entry->d_name, 0) == 0); - CHECK(closedir(entries) == 0 && rmdir(path) == 0); -} - -typedef struct { unsigned char bytes[4096]; size_t size; } Bytes; - -static bool bytes_span(void *context, PureSpan span) { - Bytes *b = context; - CHECK(b->size + span.size <= sizeof(b->bytes)); - memcpy(b->bytes + b->size, span.data, span.size); - b->size += span.size; - return true; -} - -/* Two apps of one module hold equal states: numbers bit for bit, texts and - * list items byte for byte. */ -static void same_states(const PureApp *a, const PureApp *b) { - const PureModule *m = a->module; - for (uint32_t i = 0; i < m->states; ++i) { - if (m->state_type[i] == PURE_STATE_TEXT) { - static Bytes x, y; - x.size = y.size = 0; - CHECK(pure_draft_read(a->texts[i], 0, pure_draft_size(a->texts[i]), bytes_span, &x) == PURE_OK); - CHECK(pure_draft_read(b->texts[i], 0, pure_draft_size(b->texts[i]), bytes_span, &y) == PURE_OK); - CHECK(x.size == y.size && !memcmp(x.bytes, y.bytes, x.size)); - } else if (m->state_type[i] == PURE_STATE_LIST) { - const PureRecord *r = &m->record[m->state_ref[i]]; - CHECK(pure_list_count(a->lists[i]) == pure_list_count(b->lists[i])); - for (size_t at = 0; at < pure_list_count(a->lists[i]); ++at) - for (unsigned f = 0; f < r->count; ++f) { - PureCell x = pure_list_get(a->lists[i], at, f), y = pure_list_get(b->lists[i], at, f); - CHECK(!memcmp(&x.num, &y.num, sizeof(x.num)) && x.size == y.size && (!x.size || !memcmp(x.text, y.text, x.size))); - } - } else CHECK(!memcmp(&a->values[i], &b->values[i], sizeof(double))); - } -} - -/* The same random events on an app in memory and on one whose states are - * stored and which restarts from them every few events: equal after every - * step, failed events included. */ -static void stored_equivalence(Spec spec, const char *const *words, size_t word_count, uint32_t seed) { - char *path = new_store(); - CHECK(load(spec, &module_a) == PURE_OK); - PureApp memory, stored; - CHECK(pure_app_start(&memory, &module_a, (PureData){0}) == PURE_OK); - CHECK(pure_app_start(&stored, &module_a, pure_tree_data(path)) == PURE_OK); - unsigned restarts = 0; - for (unsigned step = 0; step < 400; ++step) { - uint32_t input = next_random(&seed) % module_a.inputs; - PureResult a, b; - if (module_a.input[input].kind == PURE_KIND_TEXT) { - const char *word = words[next_random(&seed) % word_count]; - PureSpan text = {(const unsigned char *)word, strlen(word)}; - a = pure_app_text(&memory, input, text); - b = pure_app_text(&stored, input, text); - } else { - a = pure_app_input(&memory, input, PURE_EDGE_DOWN); - b = pure_app_input(&stored, input, PURE_EDGE_DOWN); - } - CHECK(a == b); - same_states(&memory, &stored); - if (next_random(&seed) % 7 == 0) { - pure_app_stop(&stored); - CHECK(pure_app_start(&stored, &module_a, pure_tree_data(path)) == PURE_OK); - same_states(&memory, &stored); - ++restarts; - } - } - CHECK(restarts > 20); - pure_app_stop(&memory); - pure_app_stop(&stored); - remove_store(path); -} - -/* One event changes a number, a text and a list; the process dies at each - * commit stage. Starting again finds all three as before or all as after. */ -enum { NOTE = 1, TASKS = 3 }; -static const S crash_nums[] = {{"n", 0}}; -static const T crash_states[] = {{"note", 1, 0}, {"at", 2, 1}, {"tasks", 3, 0}, {"i", 4, 3}}; -static const RF crash_fields[] = {{"title", 1, 0}, {"done", 0, 0}}; -static const R crash_records[] = {{"task", crash_fields, 2}}; -static const char *const crash_constants[] = {"", "x"}; -static const I crash_inputs[] = {KEY("go", "KeyG")}; -static const O crash_code[] = { - {L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, /* n = n + 1 */ - {CUL, 2, 0}, {TK, 1, 0}, {TINS, 1, 0}, /* note.insert(at, "x") */ - {TK, 1, 0}, {L, 0, 0}, {LA, 3, 0}, /* tasks.append(task { "x", n }) */ - {XE, 3, 0}, {XS, 4, 0}, /* i = tasks.last */ -}; -static const H crash_handlers[] = {{1, 0, 12}}; - -static unsigned consistent(const PureApp *app) { - unsigned n = (unsigned)app->values[0]; - CHECK(app->values[0] == n && pure_draft_size(app->texts[NOTE]) == n && app->values[2] == n); - CHECK(pure_list_count(app->lists[TASKS]) == n && app->values[4] == (n ? n - 1 : 0)); - for (unsigned k = 0; k < n; ++k) CHECK(pure_list_get(app->lists[TASKS], k, 1).num == k + 1); - return n; -} - -static void stored_crashes(void) { - char *path = new_store(); - Spec spec = {crash_nums, 1, crash_inputs, 1, NULL, 0, crash_handlers, 1, crash_code, LEN(crash_code), NULL, 0, - crash_states, 4, crash_constants, 2, NULL, 0, crash_records, 1, NULL, 0}; - CHECK(load(spec, &module_a) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); - CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK && consistent(&app) == 1); - pure_app_stop(&app); - unsigned expected = 1; - for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { - pid_t child = fork(); - CHECK(child >= 0); - if (!child) { - PureApp doomed; - if (pure_app_start(&doomed, &module_a, pure_tree_data(path)) != PURE_OK) _exit(1); - crash = 1; - fail_stage = stage; - pure_app_input(&doomed, 0, PURE_EDGE_DOWN); - _exit(0); - } - int status; - CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); - if (stage == PURE_BEFORE_DIR_SYNC) ++expected; /* the head was written */ - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == expected); - pure_app_stop(&app); - } - /* A failed commit leaves the app and the store as they were; after an - * uncertain one the app no longer matches the store and its events fail - * until it starts again, from what the store has. */ - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); - for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; ++fail_stage) { - CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_STORE_ERROR && consistent(&app) == expected); - } - fail_stage = PURE_BEFORE_DIR_SYNC; - CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_UNCERTAIN && consistent(&app) == expected); - fail_stage = -1; - CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_CONFLICT && consistent(&app) == expected); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == expected + 1); - CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK && consistent(&app) == expected + 2); - pure_app_stop(&app); - remove_store(path); -} - -/* Frame changes are stored by the next event or by a save; a stop alone - * leaves them out. */ -static void stored_frames(void) { - char *path = new_store(); - static const O code[] = {{L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}}; - static const H handlers[] = {{0, 0, 4}, {1, 0, 0}}; - Spec spec = {crash_nums, 1, crash_inputs, 1, NULL, 0, handlers, 2, code, 4, NULL, 0, NULL, 0, NULL, 0, NULL, 0, - NULL, 0, NULL, 0}; - CHECK(load(spec, &module_a) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); - for (int i = 0; i < 5; ++i) CHECK(pure_app_frame(&app) == PURE_OK); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 0); - for (int i = 0; i < 3; ++i) CHECK(pure_app_frame(&app) == PURE_OK); - CHECK(pure_app_save(&app) == PURE_OK); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 3); - for (int i = 0; i < 2; ++i) CHECK(pure_app_frame(&app) == PURE_OK); - CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 5); - pure_app_stop(&app); - remove_store(path); -} - -static bool list_keys(void *context, const char *key) { - strcat(context, key); - strcat(context, ","); - return true; -} - -/* A message's document and the states its handlers change commit together. */ -static void stored_messages(void) { - char *path = new_store(); - CHECK(load(editor_spec, &module_a) == PURE_OK); - PureApp app; - unsigned char buffer[64]; - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); - CHECK(pure_app_message(&app, 1, "doc", 0, edit_payload(buffer, 0, 0, "hello")) == PURE_OK && app.revision == 1); - CHECK(app.values[0] == 1); - CHECK(pure_app_message(&app, 1, "doc", 0, edit_payload(buffer, 0, 0, "x")) == PURE_CONFLICT && app.values[0] == 1); - /* An edit past the size guard fails: neither the document nor the count moves. */ - CHECK(pure_app_message(&app, 3, "doc", 1, edit_payload(buffer, 5, 0, "-world")) == PURE_LIMIT && app.values[0] == 1); - CHECK(pure_app_message(&app, 3, "doc", 1, edit_payload(buffer, 5, 0, "!")) == PURE_OK && app.values[0] == 2); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 2); - CHECK(pure_app_message(&app, 0, "doc", -1, (PureSpan){0}) == PURE_OK && app.revision == 2 && replied(&app, "hello!")); - pure_app_stop(&app); - /* Documents stay documents: the app's states are not among the keys. */ - char seen[256] = ""; - CHECK(pure_store_keys(path, list_keys, seen) == PURE_OK && !strcmp(seen, "doc,")); - remove_store(path); -} - -/* A stored list whose record changed keeps each field by name, when the app - * starts from it and when its module is replaced; stored values that no - * longer fit fall back, and damaged stored values stop the start. */ -static void stored_migration(void) { - char *path = new_store(); - static const RF changed_fields[] = {{"done", 0, 0}, {"note", 1, 1}, {"title", 1, 0}, {"rank", 0, 5}}; - static const R changed[] = {{"task", changed_fields, 4}}; - Spec before = {crash_nums, 1, crash_inputs, 1, NULL, 0, crash_handlers, 1, crash_code, LEN(crash_code), NULL, 0, - crash_states, 4, crash_constants, 2, NULL, 0, crash_records, 1, NULL, 0}; - Spec after = before; - after.r = changed; - after.h = NULL; - after.nh = 0; - CHECK(load(before, &module_a) == PURE_OK && load(after, &module_b) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); - CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK && pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); - pure_app_stop(&app); - for (int round = 0; round < 2; ++round) { /* migrated, then stored as migrated */ - CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK && pure_list_count(app.lists[TASKS]) == 2); - PureCell note = pure_list_get(app.lists[TASKS], 1, 1), title = pure_list_get(app.lists[TASKS], 1, 2); - CHECK(pure_list_get(app.lists[TASKS], 1, 0).num == 2 && pure_list_get(app.lists[TASKS], 1, 3).num == 5); - CHECK(note.size == 1 && note.text[0] == 'x' && title.size == 1 && title.text[0] == 'x'); - CHECK(app.values[0] == 2 && app.values[4] == 1 && pure_draft_size(app.texts[NOTE]) == 2); - pure_app_stop(&app); - } - /* Back to the first record by replacing the module; the next save stores it. */ - CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK); - CHECK(pure_app_replace(&app, &module_a) == PURE_OK && pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == 3); - pure_app_stop(&app); - /* Stored numbers that no longer fit their state start again. */ - PureDraft *values; - int64_t revision; - CHECK(pure_state_open(path, "values", &values) == PURE_OK); - unsigned char record[64] = "PVS1"; - size_t at = 4; - const char *names[] = {"at", "i", "n"}; - double stored[] = {100, 7.5, 9}; - uint8_t types[] = {PURE_STATE_CURSOR, PURE_STATE_INDEX, PURE_STATE_NUM}; - for (unsigned k = 0; k < 3; ++k) { - record[at++] = types[k]; - record[at++] = (unsigned char)strlen(names[k]); - memcpy(record + at, names[k], strlen(names[k])); - at += strlen(names[k]); - uint64_t bits; - memcpy(&bits, &stored[k], sizeof(bits)); - for (unsigned b = 0; b < 8; ++b) record[at++] = (unsigned char)(bits >> (8 * b)); - } - CHECK(pure_draft_splice(values, 0, pure_draft_size(values), (PureSpan){record, at}) == PURE_OK); - CHECK(pure_draft_commit(values, &revision) == PURE_OK); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); - CHECK(app.values[0] == 9 && app.values[2] == 0 && app.values[4] == 0); - pure_app_stop(&app); - /* Damaged stored lists and texts are refused, not replaced. */ - const char *damaged[] = {"list-tasks", "text-note"}; - for (unsigned k = 0; k < 2; ++k) { - PureDraft *draft; - CHECK(pure_state_open(path, damaged[k], &draft) == PURE_OK); - CHECK(pure_draft_splice(draft, 0, 0, (PureSpan){(const unsigned char *)"\xff", 1}) == PURE_OK); - CHECK(pure_draft_commit(draft, &revision) == PURE_OK); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_STORE_ERROR); - CHECK(pure_draft_splice(draft, 0, 1, (PureSpan){0}) == PURE_OK && pure_draft_commit(draft, &revision) == PURE_OK); - pure_draft_close(draft); - } - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); - CHECK(pure_list_count(app.lists[TASKS]) == 3 && pure_draft_size(app.texts[NOTE]) == 3 && app.values[0] == 9); - pure_app_stop(&app); - pure_draft_close(values); - remove_store(path); -} - -static void stored_states(void) { - static const char *const list_words[] = {"a", "\xea\xb0\x80", "", "\n", "\xf0\x9f\x98\x80"}; - static const char *const text_words[] = {"a", "\xea\xb0\x80", "\n", "\xf0\x9f\x98\x80", "xy\nz", ""}; - stored_equivalence(list_spec, list_words, LEN(list_words), 0x9e3779b9u); - stored_equivalence(text_spec, text_words, LEN(text_words), 0x7f4a7c15u); - stored_crashes(); - stored_frames(); - stored_messages(); - stored_migration(); -} - -/* ---- Deferred saving ---------------------------------------------------- */ - -typedef struct { uint32_t input; const char *word; } Step; - -static PureResult apply_step(PureApp *app, Step step) { - if (step.word) return pure_app_text(app, step.input, (PureSpan){(const unsigned char *)step.word, strlen(step.word)}); - return pure_app_input(app, step.input, PURE_EDGE_DOWN); -} - -/* Deferred events apply at once and last only through a save: an app that - * stops without saving comes back as it was at the last save. The oracle is - * an app in memory that replays exactly the saved events. */ -static void deferred_equivalence(Spec spec, const char *const *words, size_t word_count, uint32_t seed) { - char *path = new_store(); - CHECK(load(spec, &module_a) == PURE_OK); - static Step history[2000]; - size_t saved = 0, applied = 0; - PureApp memory, stored; - CHECK(pure_app_start(&memory, &module_a, (PureData){0}) == PURE_OK); - CHECK(pure_app_start(&stored, &module_a, pure_tree_data(path)) == PURE_OK); - CHECK(pure_app_defer(&stored, true) && !pure_app_defer(&memory, true)); - unsigned lost = 0, saves = 0; - for (unsigned step = 0; step < 600; ++step) { - Step next = {next_random(&seed) % module_a.inputs, NULL}; - if (module_a.input[next.input].kind == PURE_KIND_TEXT) next.word = words[next_random(&seed) % word_count]; - PureResult a = apply_step(&memory, next), b = apply_step(&stored, next); - CHECK(a == b); - if (a == PURE_OK) history[applied++] = next; - same_states(&memory, &stored); - uint32_t roll = next_random(&seed) % 16; - if (roll < 3) { - CHECK(pure_app_save(&stored) == PURE_OK); - saved = applied; - ++saves; - } else if (roll == 3) { - /* Stop without saving: only saved events come back. */ - pure_app_stop(&stored); - pure_app_stop(&memory); - CHECK(pure_app_start(&stored, &module_a, pure_tree_data(path)) == PURE_OK && pure_app_defer(&stored, true)); - CHECK(pure_app_start(&memory, &module_a, (PureData){0}) == PURE_OK); - for (size_t k = 0; k < saved; ++k) CHECK(apply_step(&memory, history[k]) == PURE_OK); - lost += (unsigned)(applied - saved); - applied = saved; - same_states(&memory, &stored); - } - } - CHECK(saves > 50 && lost > 0); - pure_app_stop(&memory); - pure_app_stop(&stored); - remove_store(path); -} - -/* A burst of events is saved with one commit: when that commit fails every - * event of the burst is undone, and a process that dies while saving comes - * back with all of the burst or none of it. */ -static void deferred_bursts(void) { - char *path = new_store(); - Spec spec = {crash_nums, 1, crash_inputs, 1, NULL, 0, crash_handlers, 1, crash_code, LEN(crash_code), NULL, 0, - crash_states, 4, crash_constants, 2, NULL, 0, crash_records, 1, NULL, 0}; - CHECK(load(spec, &module_a) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && pure_app_defer(&app, true)); - for (int i = 0; i < 5; ++i) CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); - CHECK(consistent(&app) == 5 && app.events == 5); - for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; fail_stage += 2) { - CHECK(pure_app_save(&app) == PURE_STORE_ERROR && consistent(&app) == 0 && app.events == 0); - for (int i = 0; i < 5; ++i) CHECK(pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_OK); - } - fail_stage = -1; - CHECK(pure_app_save(&app) == PURE_OK && consistent(&app) == 5); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == 5); - pure_app_stop(&app); - unsigned expected = 5; - for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { - pid_t child = fork(); - CHECK(child >= 0); - if (!child) { - PureApp doomed; - if (pure_app_start(&doomed, &module_a, pure_tree_data(path)) != PURE_OK || !pure_app_defer(&doomed, true)) _exit(1); - for (int i = 0; i < 7; ++i) if (pure_app_input(&doomed, 0, PURE_EDGE_DOWN) != PURE_OK) _exit(2); - crash = 1; - fail_stage = stage; - pure_app_save(&doomed); - _exit(0); - } - int status; - CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); - if (stage == PURE_BEFORE_DIR_SYNC) expected += 7; - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && consistent(&app) == expected); - pure_app_stop(&app); - } - remove_store(path); -} - -/* Messages in a burst: documents of different keys commit together with the - * states, each moving one revision; a second message for a key saves the - * burst first, and so does a full burst. */ -static void deferred_messages(void) { - char *path = new_store(); - CHECK(load(editor_spec, &module_a) == PURE_OK); - PureApp app; - unsigned char buffer[64]; - char key[8]; - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && pure_app_defer(&app, true)); - for (int i = 0; i < 3; ++i) { - snprintf(key, sizeof(key), "k%d", i); - CHECK(pure_app_message(&app, 1, key, 0, edit_payload(buffer, 0, 0, "ab")) == PURE_OK && app.revision == 1); - } - CHECK(app.values[0] == 3 && app.pending_count == 3 && app.open); - /* The same key again: the burst is saved first, then this edit joins a new one. */ - CHECK(pure_app_message(&app, 1, "k0", 1, edit_payload(buffer, 2, 0, "c")) == PURE_OK && app.revision == 2); - CHECK(app.pending_count == 1 && app.values[0] == 4); - CHECK(pure_app_message(&app, 1, "k1", 0, edit_payload(buffer, 0, 0, "x")) == PURE_CONFLICT); - CHECK(pure_app_save(&app) == PURE_OK && !app.open && !app.pending_count); - for (int i = 0; i < PURE_APP_PENDING + 1; ++i) { - snprintf(key, sizeof(key), "m%d", i); - CHECK(pure_app_message(&app, 1, key, 0, edit_payload(buffer, 0, 0, "y")) == PURE_OK); - CHECK(app.pending_count == (i < PURE_APP_PENDING ? (unsigned)i + 1 : 1)); /* full: saved on the way */ - } - pure_app_stop(&app); /* the last edit was never saved */ - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[0] == 4 + PURE_APP_PENDING); - CHECK(pure_app_message(&app, 0, "k0", -1, (PureSpan){0}) == PURE_OK && app.revision == 2 && replied(&app, "abc")); - snprintf(key, sizeof(key), "m%d", PURE_APP_PENDING); - CHECK(pure_app_message(&app, 0, key, -1, (PureSpan){0}) == PURE_NOT_FOUND); - snprintf(key, sizeof(key), "m%d", PURE_APP_PENDING - 1); - CHECK(pure_app_message(&app, 0, key, -1, (PureSpan){0}) == PURE_OK && replied(&app, "y")); - pure_app_stop(&app); - remove_store(path); -} - -static void deferred(void) { - static const char *const list_words[] = {"a", "\xea\xb0\x80", "", "\n", "\xf0\x9f\x98\x80"}; - static const char *const text_words[] = {"a", "\xea\xb0\x80", "\n", "\xf0\x9f\x98\x80", "xy\nz", ""}; - deferred_equivalence(list_spec, list_words, LEN(list_words), 0x2545f491u); - deferred_equivalence(text_spec, text_words, LEN(text_words), 0x1b873593u); - deferred_bursts(); - deferred_messages(); -} - -/* Text inserted at a cursor goes before it; at a mark (a cursor that stays), - * after it. Everywhere else, and when text is removed, both move alike. */ -static void marks(void) { - enum { MARK = 2 | PURE_CURSOR_STAYS << 8 }; - static const T textual[] = {{"note", 1, 0}, {"at", 2, 0}, {"mark", MARK, 0}}; - static const char *const constants[] = {"ab"}; - static const I inputs[] = {{"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("keep", "KeyK"), KEY("back", "Backspace")}; - static const O code[] = { - {CUL, 1, 0}, {TI, 0, 0}, {TINS, 0, 0}, /* 0 typed: at the cursor */ - {CUL, 1, 0}, {CS, 2, 0}, /* 3 keep: mark = at */ - {CUL, 1, 0}, {C, 0, -1}, {CM, 0, 0}, {CUL, 1, 0}, {TREM, 0, 0}, /* 5 back */ - }; - static const H handlers[] = {{1, 0, 3}, {3, 3, 2}, {5, 5, 5}}; - Spec spec = {NULL, 0, inputs, LEN(inputs), NULL, 0, handlers, LEN(handlers), code, LEN(code), - NULL, 0, textual, 3, constants, 1, NULL, 0, NULL, 0, NULL, 0}; - if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } - CHECK(module_a.state_flags[2] == PURE_CURSOR_STAYS && !module_a.state_flags[1]); - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"x", 1}) == PURE_OK); - CHECK(!strcmp(state_text(&app, 0), "xab") && app.values[1] == 1 && app.values[2] == 0); /* the mark stayed */ - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[2] == 1); - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"yz", 2}) == PURE_OK); - CHECK(!strcmp(state_text(&app, 0), "xyzab") && app.values[1] == 3 && app.values[2] == 1); - CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 2 && app.values[2] == 1); - CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_OK && app.values[1] == 1 && app.values[2] == 1); /* gathered */ - pure_app_stop(&app); - /* A flag only on a cursor, and only this one. */ - static const T wrong_flag[] = {{"note", 1, 0}, {"at", 2 | 2 << 8, 0}, {"mark", MARK, 0}}; - static const T flagged_text[] = {{"note", 1 | PURE_CURSOR_STAYS << 8, 0}, {"at", 2, 0}, {"mark", 2, 0}}; - Spec bad = spec; - bad.t = wrong_flag; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); - bad.t = flagged_text; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "state reference")); -} - -/* Folds: a block run once per item of a list, as a column of it, gives a - * count, a sum, a maximum or a minimum; an empty maximum has no value. */ -enum { FOLD = PURE_OP_FOLD }; -#define FOLDING(kind, list, length) ((uint32_t)(kind) | (uint32_t)(list) << 8 | (uint32_t)(length) << 16) -static const RF fold_fields[] = {{"title", 1, 0}, {"done", 0, 0}, {"points", 0, 0}}; -static const R fold_records[] = {{"task", fold_fields, 3}}; -static const T fold_states[] = {{"tasks", 3, 0}, {"at", 4, 0}}; -static const char *const fold_constants[] = {""}; -static const I fold_inputs[] = {{"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("toggle", "KeyT")}; -enum { F_TITLE = 0, F_DONE = 1u << 16, F_POINTS = 2u << 16 }; -static const O fold_code[] = { - {TI, 0, 0}, {C, 0, 0}, {TI, 0, 0}, {CHR, 0, 0}, {LA, 0, 0}, {XE, 0, 0}, {XS, 1, 0}, /* 0 typed */ - {XL, 1, 0}, {XL, 1, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, {IS, F_DONE, 0}, /* 7 toggle */ - {FOLD, FOLDING(PURE_FOLD_COUNT, 0, 3), 0}, {ROW, 0, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, /* 12 open */ - {FOLD, FOLDING(PURE_FOLD_SUM, 0, 3), 0}, {ROW, 0, 0}, {IG, F_POINTS, 0}, {C, 0, 1}, /* 16 points */ - {FOLD, FOLDING(PURE_FOLD_MAX, 0, 4), 0}, {ROW, 0, 0}, {IG, F_TITLE, 0}, {CHR, 0, 0}, /* 20 longest */ - {C, 0, 1}, - {FOLD, FOLDING(PURE_FOLD_SUM, 0, 4), 0}, {ROW, 0, 0}, {IG, F_POINTS, 0}, {ROW, 0, 0}, /* 25 done_points */ - {IG, F_DONE, 0}, - {FOLD, FOLDING(PURE_FOLD_MIN, 0, 5), 0}, {ROW, 0, 0}, {IG, F_POINTS, 0}, {ROW, 0, 0}, /* 30 fewest_open */ - {IG, F_DONE, 0}, {NOT, 0, 0}, - {C, 0, 10}, {FOLD, FOLDING(PURE_FOLD_COUNT, 0, 1), 0}, {C, 0, 1}, {ADD, 0, 0}, /* 36 count_plus */ - {FOLD, FOLDING(PURE_FOLD_FIRST, 0, 3), 0}, {ROW, 0, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, /* 40 first_open */ - {XN, 0, 0}, -}; -static const H fold_handlers[] = {{1, 0, 7}, {3, 7, 5}}; -static const V fold_views[] = {{"open", 12, 4}, {"points", 16, 4}, {"longest", 20, 5}, {"done_points", 25, 5}, - {"fewest_open", 30, 6}, {"count_plus", 36, 4}, {"first_open", 40, 5}}; -static Spec fold_spec(const O *code, size_t count, const V *views, size_t nviews, const H *handlers, size_t nhandlers) { - return (Spec){NULL, 0, fold_inputs, LEN(fold_inputs), views, nviews, handlers, nhandlers, code, count, NULL, 0, - fold_states, 2, fold_constants, 1, NULL, 0, fold_records, 1, NULL, 0}; -} - -static void folds(void) { - CHECK(load(fold_spec(fold_code, LEN(fold_code), fold_views, LEN(fold_views), fold_handlers, 2), &module_a) == PURE_OK); - PureApp app; - double value; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - CHECK(number_view(&app, 0) == 0 && number_view(&app, 1) == 0 && !pure_app_view(&app, 2, &value)); - CHECK(number_view(&app, 3) == 0 && !pure_app_view(&app, 4, &value) && number_view(&app, 5) == 10); - CHECK(!pure_app_view(&app, 6, &value)); /* no first open task yet */ - const char *const words[] = {"a", "bbb", "\xea\xb0\x80\xea\xb0\x80"}; /* one, three and two characters */ - for (size_t i = 0; i < 3; ++i) - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)words[i], strlen(words[i])}) == PURE_OK); - CHECK(number_view(&app, 0) == 3 && number_view(&app, 1) == 6 && number_view(&app, 2) == 3); - CHECK(number_view(&app, 3) == 0 && number_view(&app, 4) == 1 && number_view(&app, 5) == 13); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); /* the last task is done */ - CHECK(number_view(&app, 0) == 2 && number_view(&app, 3) == 2 && number_view(&app, 4) == 1); - CHECK(number_view(&app, 6) == 0); /* the first open one: an index */ - CHECK(pure_app_point(&app, 1, 0) == PURE_OK && pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); - CHECK(number_view(&app, 6) == 1); - CHECK(pure_app_point(&app, 1, 1) == PURE_OK && pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); - CHECK(!pure_app_view(&app, 6, &value)); - CHECK(number_view(&app, 0) == 0 && number_view(&app, 3) == 6 && !pure_app_view(&app, 4, &value)); /* none open */ - pure_app_stop(&app); - - /* What a fold may not be. */ - static const O nested[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 0, 2), 0}, {FOLD, FOLDING(PURE_FOLD_COUNT, 0, 1), 0}, {C, 0, 1}}; - static const O into[] = {{C, 0, 1}, {JZ, 3, 0}, {FOLD, FOLDING(PURE_FOLD_COUNT, 0, 1), 0}, {C, 0, 1}}; - static const O effect[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 0, 3), 0}, {ROW, 0, 0}, {XS, 1, 0}, {C, 0, 1}, - {PURE_OP_REQUIRE, 0, 0}}; - static const O short_sum[] = {{FOLD, FOLDING(PURE_FOLD_SUM, 0, 1), 0}, {C, 0, 1}}; - static const O outside[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 0, 5), 0}, {C, 0, 1}}; - static const O not_list[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 1, 1), 0}, {C, 0, 1}}; - static const struct { const O *code; size_t count, length; bool handler; const char *reason; } refused[] = { - {nested, LEN(nested), 3, false, "fold inside a fold"}, - {into, LEN(into), 4, false, "branch into a fold"}, - {effect, LEN(effect), 5, true, "fold with an effect"}, - {short_sum, LEN(short_sum), 2, false, "fold block must leave its numbers"}, - {outside, LEN(outside), 2, false, "fold block outside its block"}, - {not_list, LEN(not_list), 2, false, "no such state"}, - }; - for (size_t i = 0; i < LEN(refused); ++i) { - V view = {"v", 0, (uint32_t)refused[i].length}; - H handler = {3, 0, (uint32_t)refused[i].length}; - Spec spec = refused[i].handler ? fold_spec(refused[i].code, refused[i].count, NULL, 0, &handler, 1) - : fold_spec(refused[i].code, refused[i].count, &view, 1, NULL, 0); - if (load(spec, &module_b) != PURE_INVALID || !strstr(error, refused[i].reason)) { - fprintf(stderr, "fold case %zu: expected '%s', got '%s'\n", i, refused[i].reason, error); - abort(); - } - } -} - -/* A filtered table: one row per item its filter keeps; its columns read the - * item (ROW is the item, not the row), and hosts turn rows into items and - * back. The rows follow every change. */ -static const O filter_code[] = { - {TI, 0, 0}, {C, 0, 0}, {TI, 0, 0}, {CHR, 0, 0}, {LA, 0, 0}, {XE, 0, 0}, {XS, 1, 0}, /* 0 typed */ - {XL, 1, 0}, {XL, 1, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, {IS, F_DONE, 0}, /* 7 toggle */ - {ROW, 0, 0}, {IG, F_DONE, 0}, {NOT, 0, 0}, /* 12 filter: open */ - {ROW, 0, 0}, {IG, F_TITLE, 0}, /* 15 title */ - {ROW, 0, 0}, {XN, 0, 0}, /* 17 pos */ -}; -enum { FILTERED = 2 | 1 << 8 }; /* a table over state 0, filtered */ -static const W filter_views[] = {{"open", FILTERED, 0, 12, 3}, {"title", 1, 1, 15, 2}, {"pos", 0, 1, 17, 2}}; - -static Spec filter_spec(const O *code, size_t count, const W *views, size_t nviews) { - return (Spec){NULL, 0, fold_inputs, LEN(fold_inputs), NULL, 0, fold_handlers, 2, code, count, NULL, 0, - fold_states, 2, fold_constants, 1, NULL, 0, fold_records, 1, views, nviews}; -} - -static const char *cell_title(PureApp *app, uint32_t column, size_t row) { - static Text text; - text = (Text){{0}, 0}; - CHECK(pure_app_cell_text(app, column, row, text_span, &text) == PURE_OK && text.size < sizeof(text.text)); - text.text[text.size] = 0; - return text.text; -} - -static void filters(void) { - CHECK(load(filter_spec(filter_code, LEN(filter_code), filter_views, 3), &module_a) == PURE_OK); - PureApp app; - double value; - size_t at; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK && pure_app_rows(&app, 0) == 0); - for (const char *word = "abc"; *word; ++word) - CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)word, 1}) == PURE_OK); - CHECK(pure_app_rows(&app, 0) == 3 && !strcmp(cell_title(&app, 1, 2), "c") && pure_app_cell(&app, 2, 2, &value) && value == 2); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); /* c is done: it leaves */ - CHECK(pure_app_rows(&app, 0) == 2 && !pure_app_cell(&app, 2, 2, &value) && !pure_app_item_row(&app, 0, 2, &at)); - CHECK(pure_app_point(&app, 1, 0) == PURE_OK && pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); /* a too */ - CHECK(pure_app_rows(&app, 0) == 1 && !strcmp(cell_title(&app, 1, 0), "b")); - CHECK(pure_app_cell(&app, 2, 0, &value) && value == 1); /* the item's position, not the row's */ - CHECK(pure_app_row_item(&app, 0, 0, &at) && at == 1 && !pure_app_row_item(&app, 0, 1, &at)); - CHECK(pure_app_item_row(&app, 0, 1, &at) && at == 0 && !pure_app_item_row(&app, 0, 0, &at)); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK); /* a comes back, before b */ - CHECK(pure_app_rows(&app, 0) == 2 && !strcmp(cell_title(&app, 1, 0), "a") && pure_app_item_row(&app, 0, 1, &at) && at == 1); - CHECK(pure_app_replace(&app, &module_a) == PURE_OK && pure_app_rows(&app, 0) == 2); - pure_app_stop(&app); - - /* What a filter may not be. */ - static const O effect[] = {{ROW, 0, 0}, {XS, 1, 0}, {C, 0, 1}}; - static const O text_filter[] = {{ROW, 0, 0}, {IG, F_TITLE, 0}}; - static const O folded[] = {{FOLD, FOLDING(PURE_FOLD_COUNT, 0, 1), 0}, {C, 0, 1}}; - static const W filtered_by[] = {{"open", FILTERED, 0, 0, 0}}, open3[] = {{"open", FILTERED, 0, 0, 3}}, - open2[] = {{"open", FILTERED, 0, 0, 2}}; - static const W not_a_list[] = {{"open", 2 | 2 << 8, 0, 0, 3}}; - static const W flagged_column[] = {{"open", FILTERED, 0, 0, 3}, {"c", 0 | 1 << 8, 1, 0, 3}}; - static const struct { const O *code; size_t count; const W *views; size_t nviews; const char *reason; } refused[] = { - {effect, LEN(effect), open3, 1, "filter with an effect"}, - {text_filter, LEN(text_filter), open2, 1, "filter must leave one number"}, - {folded, LEN(folded), open2, 1, "fold inside a filter"}, - {effect, LEN(effect), filtered_by, 1, "table list"}, /* a filtered table needs its block */ - {effect, LEN(effect), not_a_list, 1, "table list"}, /* state 1 is an index */ - {effect, LEN(effect), flagged_column, 2, "view type"}, /* only a table has a list byte */ - }; - for (size_t i = 0; i < LEN(refused); ++i) - if (load(filter_spec(refused[i].code, refused[i].count, refused[i].views, refused[i].nviews), &module_b) != PURE_INVALID || - !strstr(error, refused[i].reason)) { - fprintf(stderr, "filter case %zu: expected '%s', got '%s'\n", i, refused[i].reason, error); - abort(); - } -} - -/* Damaged fold and filter modules: whatever loads runs safely. */ -static void fold_mutations(void) { - static unsigned char valid[65536], bytes[65536]; - size_t size = build(valid, fold_spec(fold_code, LEN(fold_code), fold_views, LEN(fold_views), fold_handlers, 2)); - for (unsigned kind = 0; kind < 2; ++kind) { - if (kind) size = build(valid, filter_spec(filter_code, LEN(filter_code), filter_views, 3)); - uint32_t state = kind ? 0x165667b1u : 0x27d4eb2fu; - unsigned accepted = 0; - for (unsigned round = 0; round < 20000; ++round) { - memcpy(bytes, valid, size); - for (unsigned f = 0, flips = 1 + next_random(&state) % 3; f < flips; ++f) - bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); - if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; - ++accepted; - PureApp app; - if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; - for (unsigned e = 0; e < 16; ++e) { - uint32_t input = next_random(&state) % 2; - if (module_b.input[input].kind == PURE_KIND_TEXT) pure_app_text(&app, input, (PureSpan){(const unsigned char *)"ab", 2}); - else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); - for (uint32_t v = 0; v < module_b.views; ++v) { - double value; - size_t item; - if (module_b.view_type[v] == PURE_VIEW_TABLE) { - size_t rows = pure_app_rows(&app, v); - for (size_t row = 0; row < rows + 1; ++row) pure_app_row_item(&app, v, row, &item); - } else if (module_b.view_table[v]) { - for (size_t row = 0; row < 4; ++row) - if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_cell_text(&app, v, row, ignore_span, NULL); - else pure_app_cell(&app, v, row, &value); - } else if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); - else pure_app_view(&app, v, &value); - } - } - pure_app_stop(&app); - } - printf("{\"%s_module_mutations\":20000,\"accepted_after_damage\":%u}\n", kind ? "filter" : "fold", accepted); - } -} - -/* A host points at an item: the index a view shows as its position moves - * there, as an event with no handler, saved like a key event; deferred, it - * joins the burst and a failed save undoes it with the rest. */ -static void pointing(void) { - CHECK(load(list_spec, &module_a) == PURE_OK); - CHECK(pure_module_position(&module_a, 1) == 2); /* position = at.position */ - CHECK(pure_module_position(&module_a, 0) == -1 && pure_module_position(&module_a, 2) == -1 && - pure_module_position(&module_a, 3) == -1 && pure_module_position(&module_a, 99) == -1); - char path[] = "/tmp/pure-point-XXXXXX"; - CHECK(mkdtemp(path) && pure_store_init(path) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); - CHECK(pure_app_point(&app, 2, 0) == PURE_NOT_FOUND); /* nothing to point at */ - for (int i = 0; i < 3; ++i) CHECK(pure_app_text(&app, 0, (PureSpan){(const unsigned char *)"a", 1}) == PURE_OK); - uint64_t events = app.events; - CHECK(app.values[2] == 2 && pure_app_point(&app, 2, 0) == PURE_OK && app.values[2] == 0 && app.events == events + 1); - CHECK(pure_app_point(&app, 2, 3) == PURE_NOT_FOUND && app.values[2] == 0); - CHECK(pure_app_point(&app, 0, 1) == PURE_INVALID && pure_app_point(&app, 1, 1) == PURE_INVALID); /* a number, a list */ - CHECK(pure_app_point(&app, 2, 1) == PURE_OK); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[2] == 1); - CHECK(pure_app_defer(&app, true) && pure_app_point(&app, 2, 2) == PURE_OK && app.open && app.values[2] == 2); - CHECK(pure_app_save(&app) == PURE_OK); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK && app.values[2] == 2); - pure_app_stop(&app); - remove_store(path); -} - -/* Keyed lists: items in key order (texts by bytes, numbers by value), each - * key once; found by key; a list that becomes keyed is put in order with - * its indices following their items, or refused when two items share a key. */ -enum { FIND = PURE_OP_LIST_FIND, HAS = PURE_OP_LIST_HAS, TCMP = PURE_OP_TEXT_COMPARE, - PEOPLE = 1, K_AT = 2, SCORES = 3, K_NAME = 0, K_VISITS = 1u << 16 }; -static const S keyed_nums[] = {{"n", 0}}; -static const T keyed_states[] = {{"people", 3 | 1 << 8, 0}, {"at", 4, 1}, {"scores", 3 | 1 << 8, 1}}; -static const T unkeyed_states[] = {{"people", 3, 0}, {"at", 4, 1}, {"scores", 3, 1}}; -static const RF person_fields[] = {{"name", 1, 0}, {"visits", 0, 0}}; -static const RF score_fields[] = {{"score", 0, 0}}; -static const R keyed_records[] = {{"person", person_fields, 2}, {"score", score_fields, 1}}; -static const char *const keyed_constants[] = {"", "bob", "m"}; -static const I keyed_inputs[] = { - {"typed", NULL, PURE_KIND_TEXT, false, NULL, 0}, {"seek", NULL, PURE_KIND_TEXT, false, NULL, 0}, - {"has", NULL, PURE_KIND_TEXT, false, NULL, 0}, KEY("drop", "Delete"), KEY("bump", "KeyB"), - KEY("two", "KeyN"), KEY("zero", "KeyO"), KEY("minus_zero", "KeyM"), {"twice", NULL, PURE_KIND_TEXT, false, NULL, 0}, -}; -static const O keyed_code[] = { - {TI, 0, 0}, {C, 0, 0}, {LA, PEOPLE, 0}, {TI, 0, 0}, {FIND, PEOPLE, 0}, {XS, K_AT, 0}, /* 0 typed: add, point at it */ - {TI, 0, 0}, {FIND, PEOPLE, 0}, {XS, K_AT, 0}, /* 6 seek */ - {TI, 0, 0}, {HAS, PEOPLE, 0}, {ST, 0, 0}, /* 9 has */ - {XL, K_AT, 0}, {LR, PEOPLE, 0}, /* 12 drop */ - {XL, K_AT, 0}, {XL, K_AT, 0}, {IG, PEOPLE | K_VISITS, 0}, {C, 0, 1}, {ADD, 0, 0}, /* 14 bump */ - {IS, PEOPLE | K_VISITS, 0}, - {C, 0, 2}, {LA, SCORES, 0}, {C, 0, 0}, {LA, SCORES, 0}, {C, 0, -0.0}, {LA, SCORES, 0}, /* 20 two, zero, minus_zero */ - {TI, 0, 0}, {C, 0, 0}, {LA, PEOPLE, 0}, {TI, 0, 0}, {C, 0, 0}, {LA, PEOPLE, 0}, /* 26 twice: always exists */ - {LC, PEOPLE, 0}, /* 32 view count */ - {XL, K_AT, 0}, {XN, 0, 0}, /* 33 view position */ - {XL, K_AT, 0}, {IG, PEOPLE | K_NAME, 0}, {TK, 1, 0}, {TCMP, PURE_EQUAL, 0}, /* 35 view is_bob */ - {XL, K_AT, 0}, {IG, PEOPLE | K_NAME, 0}, {TK, 2, 0}, {TCMP, PURE_LESS, 0}, /* 39 view before_m */ - {LC, SCORES, 0}, /* 43 view scores */ - {XL, K_AT, 0}, {IG, PEOPLE | K_NAME, 0}, /* 44 view current */ -}; -static const H keyed_handlers[] = {{1, 0, 6}, {3, 6, 3}, {5, 9, 3}, {7, 12, 2}, {9, 14, 6}, {11, 20, 2}, {13, 22, 2}, - {15, 24, 2}, {17, 26, 6}}; -static const V keyed_views[] = {{"count", 32, 1}, {"position", 33, 2}, {"is_bob", 35, 4}, {"before_m", 39, 4}, - {"scores_count", 43, 1}}; -static const V keyed_text_views[] = {{"current", 44, 2}}; -/* Unkeyed, the same states: typed appends and points at the new last item. */ -static const O unkeyed_code[] = {{TI, 0, 0}, {C, 0, 0}, {LA, PEOPLE, 0}, {XE, PEOPLE, 0}, {XS, K_AT, 0}}; -static const H unkeyed_handlers[] = {{1, 0, 5}}; - -static Spec keyed_spec(const T *states, const O *code, size_t count, const H *handlers, size_t nhandlers) { - bool views = code == keyed_code; - return (Spec){keyed_nums, 1, keyed_inputs, LEN(keyed_inputs), keyed_views, views ? LEN(keyed_views) : 0, handlers, - nhandlers, code, count, NULL, 0, states, LEN(keyed_states), keyed_constants, LEN(keyed_constants), - keyed_text_views, views ? 1 : 0, keyed_records, 2, NULL, 0}; -} - -static PureResult type_text(PureApp *app, uint32_t input, const char *text) { - return pure_app_text(app, input, (PureSpan){(const unsigned char *)text, strlen(text)}); -} - -/* A list's first field (text) of every item, comma separated. */ -static const char *names_of(const PureApp *app, uint32_t list) { - static char out[512]; - size_t used = 0; - for (size_t at = 0; at < pure_list_count(app->lists[list]); ++at) { - PureCell cell = pure_list_get(app->lists[list], at, 0); - CHECK(used + cell.size + 2 < sizeof(out)); - if (at) out[used++] = ','; - if (cell.size) memcpy(out + used, cell.text, cell.size); - used += cell.size; - } - out[used] = 0; - return out; -} - -static void keyed(void) { - CHECK(load(keyed_spec(keyed_states, keyed_code, LEN(keyed_code), keyed_handlers, LEN(keyed_handlers)), &module_a) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - for (const char *const *word = (const char *const[]){"m", "b", "z", "a", NULL}; *word; ++word) - CHECK(type_text(&app, 0, *word) == PURE_OK); - CHECK(!strcmp(names_of(&app, PEOPLE), "a,b,m,z") && app.values[K_AT] == 0 && !strcmp(view_text(&app, 5), "a")); - /* A key once: the add fails, and so does an event whose second add repeats its first. */ - CHECK(type_text(&app, 0, "b") == PURE_CONFLICT && app.fault == PURE_FAULT_EXISTS); - CHECK(type_text(&app, 8, "c") == PURE_CONFLICT && !strcmp(names_of(&app, PEOPLE), "a,b,m,z") && app.values[K_AT] == 0); - CHECK(type_text(&app, 1, "z") == PURE_OK && app.values[K_AT] == 3 && !strcmp(view_text(&app, 5), "z")); - CHECK(type_text(&app, 1, "q") == PURE_NOT_FOUND && app.fault == PURE_FAULT_NOT_FOUND && app.values[K_AT] == 3); - CHECK(type_text(&app, 2, "m") == PURE_OK && app.values[0] == 1); - CHECK(type_text(&app, 2, "c") == PURE_OK && app.values[0] == 0); - CHECK(pure_app_input(&app, 4, PURE_EDGE_DOWN) == PURE_OK && pure_list_get(app.lists[PEOPLE], 3, 1).num == 1); - CHECK(pure_app_input(&app, 3, PURE_EDGE_DOWN) == PURE_OK && !strcmp(names_of(&app, PEOPLE), "a,b,m")); - CHECK(app.values[K_AT] == 2 && number_view(&app, 2) == 0 && number_view(&app, 3) == 0); /* "m": not bob, not before m */ - CHECK(type_text(&app, 1, "b") == PURE_OK && number_view(&app, 2) == 0 && number_view(&app, 3) == 1); - CHECK(type_text(&app, 0, "bob") == PURE_OK && number_view(&app, 2) == 1 && number_view(&app, 3) == 1); - /* Bytes order texts: a text before the longer ones it begins, UTF-8 after ASCII, the empty text first. */ - CHECK(type_text(&app, 0, "\xea\xb0\x80") == PURE_OK && type_text(&app, 0, "ab") == PURE_OK); - CHECK(type_text(&app, 0, "") == PURE_OK && type_text(&app, 0, "") == PURE_CONFLICT); - CHECK(!strcmp(names_of(&app, PEOPLE), ",a,ab,b,bob,m,\xea\xb0\x80") && app.values[K_AT] == 0); - /* Numbers order by value, and -0 is the key 0. */ - CHECK(pure_app_input(&app, 5, PURE_EDGE_DOWN) == PURE_OK && pure_app_input(&app, 6, PURE_EDGE_DOWN) == PURE_OK); - CHECK(pure_app_input(&app, 7, PURE_EDGE_DOWN) == PURE_CONFLICT && number_view(&app, 4) == 2); - CHECK(pure_list_get(app.lists[SCORES], 0, 0).num == 0 && !signbit(pure_list_get(app.lists[SCORES], 0, 0).num) && - pure_list_get(app.lists[SCORES], 1, 0).num == 2); - - /* Replacing: unkeyed, the list stays as it is and takes appends at its end. */ - CHECK(load(keyed_spec(unkeyed_states, unkeyed_code, LEN(unkeyed_code), unkeyed_handlers, 1), &module_b) == PURE_OK); - CHECK(pure_app_replace(&app, &module_b) == PURE_OK && type_text(&app, 0, "c") == PURE_OK); - CHECK(!strcmp(names_of(&app, PEOPLE), ",a,ab,b,bob,m,\xea\xb0\x80,c") && app.values[K_AT] == 7); - CHECK(pure_app_point(&app, K_AT, 6) == PURE_OK); /* at the Korean name */ - /* Keyed again: put in order, the index following its item. */ - CHECK(pure_app_replace(&app, &module_a) == PURE_OK && !strcmp(names_of(&app, PEOPLE), ",a,ab,b,bob,c,m,\xea\xb0\x80")); - CHECK(app.values[K_AT] == 7 && !strcmp(view_text(&app, 5), "\xea\xb0\x80")); - /* Two items with one key cannot become keyed: the app keeps its module and items. */ - CHECK(pure_app_replace(&app, &module_b) == PURE_OK && type_text(&app, 0, "a") == PURE_OK); - CHECK(pure_app_replace(&app, &module_a) == PURE_CONFLICT && app.module == &module_b && pure_list_count(app.lists[PEOPLE]) == 9); - pure_app_stop(&app); - - /* Stored: a list that becomes keyed is stored in order, once. */ - char *path = new_store(); - CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK); - for (const char *const *word = (const char *const[]){"m", "b", "z", NULL}; *word; ++word) - CHECK(type_text(&app, 0, *word) == PURE_OK); - CHECK(pure_app_point(&app, K_AT, 1) == PURE_OK); - pure_app_stop(&app); - for (int round = 0; round < 2; ++round) { - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_OK); - CHECK(!strcmp(names_of(&app, PEOPLE), "b,m,z") && app.values[K_AT] == 0); - pure_app_stop(&app); - } - CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK && type_text(&app, 0, "m") == PURE_OK); - pure_app_stop(&app); - CHECK(pure_app_start(&app, &module_a, pure_tree_data(path)) == PURE_CONFLICT); - CHECK(pure_app_start(&app, &module_b, pure_tree_data(path)) == PURE_OK && !strcmp(names_of(&app, PEOPLE), "b,m,z,m")); - pure_app_stop(&app); - remove_store(path); - static const char *const words[] = {"a", "b", "ab", "", "\xea\xb0\x80", "zz"}; - for (uint32_t seed = 1; seed <= 3; ++seed) - stored_equivalence(keyed_spec(keyed_states, keyed_code, LEN(keyed_code), keyed_handlers, LEN(keyed_handlers)), - words, LEN(words), seed * 0x9e3779b9u); - - /* What a keyed list and a text comparison may not be. */ - static const O insert[] = {{XF, PEOPLE, 0}, {TK, 0, 0}, {C, 0, 0}, {LI, PEOPLE, 0}}; - static const O set_key[] = {{XL, K_AT, 0}, {TK, 0, 0}, {IS, PEOPLE | K_NAME, 0}}; - static const O find_number[] = {{C, 0, 1}, {FIND, PEOPLE, 0}, {XS, K_AT, 0}}; - static const O find[] = {{TI, 0, 0}, {FIND, PEOPLE, 0}, {XS, K_AT, 0}}; - static const O relation[] = {{TI, 0, 0}, {TI, 0, 0}, {TCMP, PURE_NOT_EQUAL + 1, 0}, {ST, 0, 0}}; - static const O compare_numbers[] = {{C, 0, 1}, {C, 0, 1}, {TCMP, PURE_EQUAL, 0}, {ST, 0, 0}}; - static const T far_key[] = {{"people", 3 | 3 << 8, 0}, {"at", 4, 1}, {"scores", 3, 1}}; - static const T keyed_index[] = {{"people", 3, 0}, {"at", 4 | 1 << 8, 1}, {"scores", 3, 1}}; - static const struct { const T *states; const O *code; size_t count; const char *reason; } refused[] = { - {keyed_states, insert, LEN(insert), "insert by position into a keyed list"}, - {keyed_states, set_key, LEN(set_key), "change of a keyed list's key"}, - {keyed_states, find_number, LEN(find_number), "operand type"}, - {unkeyed_states, find, LEN(find), "no such state"}, - {keyed_states, relation, LEN(relation), "no such state"}, - {keyed_states, compare_numbers, LEN(compare_numbers), "operand type"}, - {far_key, unkeyed_code, LEN(unkeyed_code), "state reference"}, - {keyed_index, unkeyed_code, LEN(unkeyed_code), "state reference"}, - }; - for (size_t i = 0; i < LEN(refused); ++i) { - H handler = {1, 0, (uint32_t)refused[i].count}; - if (load(keyed_spec(refused[i].states, refused[i].code, refused[i].count, &handler, 1), &module_b) != PURE_INVALID || - !strstr(error, refused[i].reason)) { - fprintf(stderr, "keyed case %zu: expected '%s', got '%s'\n", i, refused[i].reason, error); - abort(); - } - } -} - -/* Damaged keyed modules: whatever loads keeps every keyed list in strict key - * order and every index on an item. */ -static void keyed_mutations(void) { - static unsigned char valid[65536], bytes[65536]; - size_t size = build(valid, keyed_spec(keyed_states, keyed_code, LEN(keyed_code), keyed_handlers, LEN(keyed_handlers))); - static const char *const words[] = {"a", "b", "ab", "", "\xea\xb0\x80"}; - uint32_t state = 0x6c8e9cf5u; - unsigned accepted = 0; - for (unsigned round = 0; round < 20000; ++round) { - memcpy(bytes, valid, size); - for (unsigned f = 0, flips = 1 + next_random(&state) % 3; f < flips; ++f) - bytes[next_random(&state) % size] ^= (unsigned char)(1u << (next_random(&state) % 8)); - if (pure_module_load(bytes, size, &module_b, error, sizeof(error)) != PURE_OK) continue; - ++accepted; - PureApp app; - if (pure_app_start(&app, &module_b, (PureData){0}) != PURE_OK) continue; - for (unsigned e = 0; e < 24; ++e) { - uint32_t input = next_random(&state) % module_b.inputs; - if (module_b.input[input].kind == PURE_KIND_TEXT) type_text(&app, input, words[next_random(&state) % LEN(words)]); - else if (module_b.input[input].kind == PURE_KIND_KEY) pure_app_input(&app, input, PURE_EDGE_DOWN); - for (uint32_t v = 0; v < module_b.views; ++v) { - double value; - if (module_b.view_type[v] == PURE_VIEW_TEXT) pure_app_view_text(&app, v, ignore_span, NULL); - else pure_app_view(&app, v, &value); - } - for (uint32_t s = 0; s < module_b.states; ++s) { - if (module_b.state_type[s] == PURE_STATE_INDEX) { - size_t count = pure_list_count(app.lists[module_b.state_ref[s]]); - CHECK(app.values[s] == floor(app.values[s]) && app.values[s] >= 0 && - app.values[s] <= (count ? (double)(count - 1) : 0)); - } - if (module_b.state_type[s] != PURE_STATE_LIST || !module_b.state_key[s]) continue; - unsigned key = module_b.state_key[s] - 1u; - bool text = module_b.record_field_text[module_b.record[module_b.state_ref[s]].first + key]; - for (size_t at = 1; at < pure_list_count(app.lists[s]); ++at) { - PureCell a = pure_list_get(app.lists[s], at - 1, key), b = pure_list_get(app.lists[s], at, key); - if (!text) { CHECK(a.num < b.num); continue; } - size_t shorter = a.size < b.size ? a.size : b.size; - int order = shorter ? memcmp(a.text, b.text, shorter) : 0; - CHECK(order < 0 || (!order && a.size < b.size)); - } - } - } - pure_app_stop(&app); - } - printf("{\"keyed_module_mutations\":20000,\"accepted_after_damage\":%u}\n", accepted); -} - -/* ---- Snapshots. Everything an app's states and views show, as text (safe - * on any thread): a reader of a snapshot must show what the app showed. */ -typedef struct { char *text; size_t used, size; } Shown; - -static bool show_span(void *context, PureSpan span) { - Shown *s = context; - CHECK(s->used + span.size < s->size); - if (span.size) memcpy(s->text + s->used, span.data, span.size); - s->used += span.size; - return true; -} - -static void show(Shown *s, const char *format, double value) { - int n = snprintf(s->text + s->used, s->size - s->used, format, value); - CHECK(n >= 0 && (size_t)n < s->size - s->used); - s->used += (size_t)n; -} - -static void describe(PureApp *app, Shown *s) { - const PureModule *m = app->module; - s->used = 0; - for (uint32_t i = 0; i < m->states; ++i) { - if (m->state_type[i] == PURE_STATE_TEXT) - CHECK(pure_draft_read(app->texts[i], 0, pure_draft_size(app->texts[i]), show_span, s) == PURE_OK); - else if (m->state_type[i] == PURE_STATE_LIST) { - const PureRecord *r = &m->record[m->state_ref[i]]; - for (size_t at = 0; at < pure_list_count(app->lists[i]); ++at) - for (unsigned f = 0; f < r->count; ++f) { - PureCell cell = pure_list_get(app->lists[i], at, f); - if (m->record_field_text[r->first + f]) show_span(s, (PureSpan){cell.text, cell.size}); - else show(s, "%g", cell.num); - show(s, f + 1 < r->count ? "," : ";", 0); - } - } else show(s, "%g", app->values[i]); - show(s, "|", 0); - } - for (uint32_t v = 0; v < m->views; ++v) { - double value; - if (m->view_table[v] || m->view_type[v] == PURE_VIEW_TABLE) continue; - if (m->view_type[v] == PURE_VIEW_TEXT) { - if (pure_app_view_text(app, v, show_span, s) != PURE_OK) show(s, "!", 0); - } else if (pure_app_view(app, v, &value)) show(s, "%g", value); - else show(s, "!", 0); - show(s, "|", 0); - } - s->text[s->used] = 0; -} - -/* One random event of the keyed app (see keyed), saved as it returns. */ -static void keyed_event(PureApp *app, uint32_t *seed) { - static const char *const names[] = {"a", "b", "ab", "", "\xea\xb0\x80", "zz", "bob", "m", "q", "c"}; - uint32_t input = next_random(seed) % app->module->inputs; - if (app->module->input[input].kind == PURE_KIND_TEXT) type_text(app, input, names[next_random(seed) % LEN(names)]); - else pure_app_input(app, input, PURE_EDGE_DOWN); -} - -/* A snapshot shows the states of the save before it, as they were, however - * the app goes on (items it changes or drops, a module it switches to, even - * after it stops and its store closes); a change not saved yet refuses one, - * and a save that fails is never shown. */ -static size_t snapshot_retired, snapshot_reads; - -static void snapshots_kept(Spec spec, const Spec *other, void (*event)(PureApp *, uint32_t *), unsigned steps) { - enum { KEPT = 8, SIZE = 1 << 16 }; - static char texts[KEPT + 2][SIZE]; - char *path = new_store(); - PureStore *store; - CHECK(load(spec, &module_a) == PURE_OK && pure_store_open(path, &store) == PURE_OK); - CHECK(!other || load(*other, &module_b) == PURE_OK); - PureApp app, reader = {0}; - CHECK(pure_app_start(&app, &module_a, pure_store_data(store)) == PURE_OK); - PureSnapshot *kept[KEPT] = {NULL}; - Shown seen[KEPT], now = {texts[KEPT], 0, SIZE}, shown = {texts[KEPT + 1], 0, SIZE}; - for (unsigned k = 0; k < KEPT; ++k) seen[k] = (Shown){texts[k], 0, SIZE}; - uint32_t seed = 0x51a95u; - for (unsigned step = 0; step < steps; ++step) { - /* Late on, another module (the keyed app unkeyed), and back if it can. */ - if (other && (step == steps - 400 || step == steps - 300)) - CHECK(pure_app_replace(&app, app.module == &module_a ? &module_b : &module_a) != PURE_INVALID); - event(&app, &seed); - snapshot_retired += app.retired.count; - PureSnapshot *made = pure_app_snapshot(&app); - CHECK(made); - describe(&app, &now); - pure_app_read(&reader, made); - describe(&reader, &shown); - CHECK(!strcmp(now.text, shown.text)); - unsigned slot = next_random(&seed) % KEPT; - if (kept[slot]) { - pure_app_read(&reader, kept[slot]); - describe(&reader, &shown); - CHECK(!strcmp(seen[slot].text, shown.text)); - pure_snapshot_release(kept[slot]); - } - kept[slot] = made; - memcpy(seen[slot].text, now.text, now.used + 1); - } - /* Deferred: an unsaved change refuses a snapshot, and a failed save - * leaves the one before it true. */ - CHECK(pure_app_defer(&app, true)); - describe(&app, &now); - PureResult saved = PURE_OK; - for (unsigned tries = 0; tries < 64 && saved == PURE_OK; ++tries) { - event(&app, &seed); - if (!app.open) continue; - CHECK(!pure_app_snapshot(&app)); - fail_stage = PURE_AFTER_HEADER; - saved = pure_app_save(&app); /* OK when the event changed nothing */ - fail_stage = -1; - } - CHECK(saved == PURE_STORE_ERROR); - PureSnapshot *made = pure_app_snapshot(&app); - CHECK(made); - pure_app_read(&reader, made); - describe(&reader, &shown); - CHECK(!strcmp(now.text, shown.text)); - pure_snapshot_release(made); - /* The kept ones outlive the app and its store. */ - pure_app_stop(&app); - pure_store_close(store); - for (unsigned k = 0; k < KEPT; ++k) { - pure_app_read(&reader, kept[k]); - describe(&reader, &shown); - CHECK(!strcmp(seen[k].text, shown.text)); - pure_snapshot_release(kept[k]); - } - pure_app_stop(&reader); - remove_store(path); -} - -/* Readers on two threads take the newest snapshot while the app applies - * events and makes one after each save: each shows what the app showed. */ -typedef struct { - pthread_mutex_t lock; - PureSnapshot *newest; - char *seen; - bool done; - unsigned reads; -} Published; - -static void *read_newest(void *context) { - Published *p = context; - PureApp reader = {0}; - static _Thread_local char a[1 << 16], b[1 << 16]; - Shown expected = {a, 0, sizeof(a)}, shown = {b, 0, sizeof(b)}; - for (bool done = false; !done;) { - pthread_mutex_lock(&p->lock); - PureSnapshot *s = p->newest; - if (s) { - pure_snapshot_hold(s); - strcpy(expected.text, p->seen); - } - done = p->done; - pthread_mutex_unlock(&p->lock); - if (!s) continue; - pure_app_read(&reader, s); - describe(&reader, &shown); - CHECK(!strcmp(expected.text, shown.text)); - pure_snapshot_release(s); - __atomic_add_fetch(&p->reads, 1, __ATOMIC_RELAXED); - } - pure_app_stop(&reader); - return NULL; -} - -static void snapshots_read(Spec spec, void (*event)(PureApp *, uint32_t *), unsigned steps) { - static char seen[1 << 16], now[1 << 16]; - char *path = new_store(); - PureStore *store; - CHECK(load(spec, &module_a) == PURE_OK && pure_store_open(path, &store) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, pure_store_data(store)) == PURE_OK); - Published p = {.lock = PTHREAD_MUTEX_INITIALIZER, .seen = seen}; - pthread_t readers[2]; - for (unsigned r = 0; r < 2; ++r) CHECK(pthread_create(&readers[r], NULL, read_newest, &p) == 0); - Shown shown = {now, 0, sizeof(now)}; - uint32_t seed = 0x7eadu; - for (unsigned step = 0; step < steps; ++step) { - event(&app, &seed); - PureSnapshot *made = pure_app_snapshot(&app); - CHECK(made); - describe(&app, &shown); - pthread_mutex_lock(&p.lock); - PureSnapshot *old = p.newest; - p.newest = made; - memcpy(seen, shown.text, shown.used + 1); - pthread_mutex_unlock(&p.lock); - pure_snapshot_release(old); - } - pthread_mutex_lock(&p.lock); - p.done = true; - pthread_mutex_unlock(&p.lock); - for (unsigned r = 0; r < 2; ++r) CHECK(pthread_join(readers[r], NULL) == 0); - CHECK(p.reads > steps / 4); - snapshot_reads += p.reads; - pure_snapshot_release(p.newest); - pure_app_stop(&app); - pure_store_close(store); - remove_store(path); -} - -static void crash_event(PureApp *app, uint32_t *seed) { - (void)seed; - CHECK(pure_app_input(app, 0, PURE_EDGE_DOWN) == PURE_OK); -} - -static void snapshots(void) { - Spec keyed = keyed_spec(keyed_states, keyed_code, LEN(keyed_code), keyed_handlers, LEN(keyed_handlers)); - Spec crashing = {crash_nums, 1, crash_inputs, 1, NULL, 0, crash_handlers, 1, crash_code, LEN(crash_code), NULL, 0, - crash_states, 4, crash_constants, 2, NULL, 0, crash_records, 1, NULL, 0}; - Spec unkeyed = keyed_spec(unkeyed_states, unkeyed_code, LEN(unkeyed_code), unkeyed_handlers, 1); - snapshots_kept(keyed, &unkeyed, keyed_event, 3000); - snapshots_kept(crashing, NULL, crash_event, 300); - snapshots_read(keyed, keyed_event, 3000); - snapshots_read(crashing, crash_event, 300); - /* Without stored states (texts only in memory) there is no snapshot. */ - CHECK(load(crashing, &module_a) == PURE_OK); - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK && !pure_app_snapshot(&app)); - pure_app_stop(&app); - CHECK(snapshot_retired > 100); - printf("{\"snapshots\":6600,\"items_retired\":%zu,\"reads_on_threads\":%zu}\n", snapshot_retired, snapshot_reads); -} - -/* A message of a text, a number, bytes and a u32, in that order: the prefix - * holds the text's length, the number and the u32; the text's bytes come - * first after it and the bytes (the last variable field) take the rest. A - * length past the payload, a text that is not UTF-8 or a number that is not - * finite fails the message. */ -enum { TSET = PURE_OP_TEXT_SET }; -static void message_fields(void) { - static const S nums[] = {{"n", 0}}; - static const T texts[] = {{"title", 1, 0}}; - static const char *const constants[] = {""}; - static const F fields[] = {{"title", PURE_FIELD_TEXT}, {"points", PURE_FIELD_NUMBER}, {"note", PURE_FIELD_BYTES}, - {"count", PURE_FIELD_U32}}; - static const I inputs[] = {{"post", NULL, PURE_KIND_MESSAGE, false, fields, 4}}; - static const O code[] = {{FIELD, 1, 0}, {FIELD, 3, 0}, {ADD, 0, 0}, {ST, 0, 0}, {FIELD, 0, 0}, {TSET, 1, 0}, - {FIELD, 2, 0}, {REPLY, 0, 0}}; - static const H handlers[] = {{1, 0, 8}}; - Spec spec = {nums, 1, inputs, 1, NULL, 0, handlers, 1, code, LEN(code), NULL, 0, texts, 1, constants, 1, NULL, 0, - NULL, 0, NULL, 0}; - if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } - CHECK(module_a.input[0].prefix == 16 && module_a.field[0].offset == 0 && module_a.field[1].offset == 4 && - module_a.field[2].offset == PURE_FIELD_REST && module_a.field[3].offset == 12); - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - unsigned char payload[64]; - /* title "가나" (6 bytes), points 2.5, count 3, then the title's bytes and note "xyz". */ - size_t size = 0; - uint32_t title_size = 6, count = 3; - double points = 2.5; - memcpy(payload, &title_size, 4); - memcpy(payload + 4, &points, 8); - memcpy(payload + 12, &count, 4); - memcpy(payload + 16, "\xea\xb0\x80\xeb\x82\x98" "xyz", 9); - size = 25; - CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, size}) == PURE_OK); - CHECK(app.values[0] == 5.5 && !strcmp(state_text(&app, 1), "\xea\xb0\x80\xeb\x82\x98") && replied(&app, "xyz")); - pure_app_finish(&app); - /* Empty variable fields. */ - title_size = 0; - memcpy(payload, &title_size, 4); - CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 16}) == PURE_OK && !*state_text(&app, 1) && replied(&app, "")); - pure_app_finish(&app); - /* A title longer than what is there, a title that is not UTF-8, a number that is not finite, a short prefix. */ - title_size = 10; - memcpy(payload, &title_size, 4); - CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 25}) == PURE_LIMIT && app.fault == PURE_FAULT_LIMIT); - title_size = 2; - memcpy(payload, &title_size, 4); - memcpy(payload + 16, "\xff\xfe", 2); - CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 18}) == PURE_INVALID && app.fault == PURE_FAULT_ENCODING); - title_size = 0; - points = NAN; - memcpy(payload, &title_size, 4); - memcpy(payload + 4, &points, 8); - CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 16}) == PURE_INVALID && app.fault == PURE_FAULT_NON_FINITE); - CHECK(pure_app_message(&app, 0, NULL, -1, (PureSpan){payload, 15}) == PURE_LIMIT); - CHECK(app.values[0] == 5.5 && !*state_text(&app, 1)); /* failed messages changed nothing */ - pure_app_stop(&app); - /* A text field is text and a bytes field bytes. */ - static const O text_as_bytes[] = {{FIELD, 0, 0}, {LEN_, 0, 0}, {ST, 0, 0}}; - static const O bytes_as_text[] = {{FIELD, 2, 0}, {TSET, 1, 0}}; - static const H three[] = {{1, 0, 3}}, two[] = {{1, 0, 2}}; - Spec bad = spec; - bad.o = text_as_bytes; bad.no = 3; bad.h = three; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); - bad.o = bytes_as_text; bad.no = 2; bad.h = two; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "operand type")); - - /* Placed by the host: the variable fields wherever they are in the - * payload (here a JSON body after the prefix), in any order; the - * prefix's lengths are not read. */ - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - static const char body[] = "{\"note\":\"xyz\",\"title\":\"\xea\xb0\x80\xeb\x82\x98\"}"; - points = 2.5; - title_size = 999; - memcpy(payload, &title_size, 4); - memcpy(payload + 4, &points, 8); - memcpy(payload + 12, &count, 4); - memcpy(payload + 16, body, sizeof(body) - 1); - size = 16 + sizeof(body) - 1; - uint32_t placed[8] = {16 + 23, 6, 0, 0, 16 + 9, 3, 0, 0}; - CHECK(pure_app_message_fields(&app, 0, NULL, -1, (PureSpan){payload, size}, placed) == PURE_OK); - CHECK(app.values[0] == 5.5 && !strcmp(state_text(&app, 1), "\xea\xb0\x80\xeb\x82\x98") && replied(&app, "xyz")); - pure_app_finish(&app); - /* A field that starts or ends past the payload, or a title that is not UTF-8 there. */ - placed[0] = (uint32_t)size + 1; - CHECK(pure_app_message_fields(&app, 0, NULL, -1, (PureSpan){payload, size}, placed) == PURE_LIMIT); - placed[0] = (uint32_t)size - 2; - placed[1] = 3; - CHECK(pure_app_message_fields(&app, 0, NULL, -1, (PureSpan){payload, size}, placed) == PURE_LIMIT); - placed[0] = 16 + 24; - placed[1] = 2; - CHECK(pure_app_message_fields(&app, 0, NULL, -1, (PureSpan){payload, size}, placed) == PURE_INVALID && - app.fault == PURE_FAULT_ENCODING); - CHECK(app.values[0] == 5.5); - pure_app_stop(&app); - - /* Fields in groups: a name is a path of identifiers (user.name), and - * never both a field and a group. */ - static const F grouped[] = {{"user.name", PURE_FIELD_TEXT}, {"user.age", PURE_FIELD_U32}, {"plan", PURE_FIELD_TEXT}}; - static const I grouped_inputs[] = {{"post", NULL, PURE_KIND_MESSAGE, false, grouped, 3}}; - static const O grouped_code[] = {{FIELD, 1, 0}, {ST, 0, 0}, {FIELD, 0, 0}, {TSET, 1, 0}}; - static const H grouped_handlers[] = {{1, 0, 4}}; - Spec groups = spec; - groups.in = grouped_inputs; groups.o = grouped_code; groups.no = LEN(grouped_code); groups.h = grouped_handlers; - CHECK(load(groups, &module_b) == PURE_OK && !strcmp(module_b.field[0].name, "user.name")); - static const char *const refused_paths[] = {"user..name", ".user", "user.", "user.1st", "user name"}; - for (size_t i = 0; i < LEN(refused_paths); ++i) { - F renamed[3] = {grouped[0], grouped[1], grouped[2]}; - renamed[0].name = refused_paths[i]; - I renamed_inputs[1] = {{"post", NULL, PURE_KIND_MESSAGE, false, renamed, 3}}; - groups.in = renamed_inputs; - CHECK(load(groups, &module_b) == PURE_INVALID && strstr(error, "message field")); - } - F both[3] = {{"user", PURE_FIELD_TEXT}, grouped[1], grouped[2]}; - I both_inputs[1] = {{"post", NULL, PURE_KIND_MESSAGE, false, both, 3}}; - groups.in = both_inputs; - CHECK(load(groups, &module_b) == PURE_INVALID && strstr(error, "field and group")); -} - -/* ---- Connections between apps (docs/research/078-app-connections.md): a - * send is part of its event and is seen once saved; a receiver applies each - * numbered message once, counts a refused one as heard, and forgets what a - * failed save undid. */ -enum { SEND = PURE_OP_SEND }; -typedef struct { uint32_t output; uint64_t seq; unsigned char payload[64]; size_t size; } Sent; -typedef struct { Sent items[16]; size_t count; } Outbox; - -static bool collect_sent(void *context, uint32_t output, uint64_t seq, PureSpan payload) { - Outbox *o = context; - CHECK(o->count < 16 && payload.size <= 64); - Sent *s = &o->items[o->count++]; - *s = (Sent){output, seq, {0}, payload.size}; - memcpy(s->payload, payload.data, payload.size); - return true; -} - -static Outbox outbox_of(const PureApp *app) { - Outbox o = {.count = 0}; - pure_app_outbox(app, collect_sent, &o); - return o; -} - -/* buy/fail {item: text, count: u32}: the count at 0, then the item. */ -static PureResult buy(PureApp *app, uint32_t input, const char *item, uint32_t count) { - unsigned char payload[64]; - size_t length = strlen(item); - memcpy(payload, &count, 4); - memcpy(payload + 4, item, length); - PureResult result = pure_app_message(app, input, NULL, -1, (PureSpan){payload, 4 + length}); - pure_app_finish(app); - return result; -} - -/* order {item: text, count: u32, total: number}: count, total, then item. */ -static PureResult deliver(PureApp *app, const char *source, uint64_t seq, const char *item, uint32_t count) { - unsigned char payload[64]; - size_t length = strlen(item); - double total = 0; - memcpy(payload, &count, 4); - memcpy(payload + 4, &total, 8); - memcpy(payload + 12, item, length); - PureResult result = pure_app_deliver(app, 0, source, seq, (PureSpan){payload, 12 + length}, NULL); - pure_app_finish(app); - return result; -} - -static void links(void) { - static const S sender_nums[] = {{"n", 0}}; - static const F order_fields[] = {{"item", PURE_FIELD_TEXT}, {"count", PURE_FIELD_U32}}; - static const F ordered_fields[] = {{"item", PURE_FIELD_TEXT}, {"count", PURE_FIELD_U32}, {"total", PURE_FIELD_NUMBER}}; - static const I sender_inputs[] = {{"buy", NULL, PURE_KIND_MESSAGE, false, order_fields, 2}, - {"fail", NULL, PURE_KIND_MESSAGE, false, order_fields, 2}, - {"ordered", NULL, PURE_KIND_OUTPUT, false, ordered_fields, 3}}; - static const O sender_code[] = {{L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}, {FIELD, 0, 0}, {FIELD, 1, 0}, - {L, 0, 0}, {SEND, 2, 0}, /* buy */ - {FIELD, 0, 0}, {FIELD, 1, 0}, {C, 0, 0}, {SEND, 2, 0}, {C, 0, 0}, {REQ, 0, 0}}; /* fail */ - static const H sender_handlers[] = {{1, 0, 8}, {3, 8, 6}}; - Spec sender = {sender_nums, 1, sender_inputs, 3, NULL, 0, sender_handlers, 2, sender_code, LEN(sender_code), - NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; - static const S receiver_nums[] = {{"stock", 0}}; - static const T receiver_texts[] = {{"last", 1, 0}}; - static const char *const receiver_constants[] = {""}; - static const I receiver_inputs[] = {{"order", NULL, PURE_KIND_MESSAGE, false, ordered_fields, 3}}; - static const O receiver_code[] = {{FIELD, 1, 0}, {L, 0, 0}, {ADD, 0, 0}, {ST, 0, 0}, {FIELD, 0, 0}, {TSET, 1, 0}, - {FIELD, 1, 0}, {C, 0, 100}, {LT, 0, 0}, {REQ, 0, 0}}; /* refuses 100 or more */ - static const H receiver_handlers[] = {{1, 0, 10}}; - Spec receiver = {receiver_nums, 1, receiver_inputs, 1, NULL, 0, receiver_handlers, 1, receiver_code, - LEN(receiver_code), NULL, 0, receiver_texts, 1, receiver_constants, 1, NULL, 0, NULL, 0, NULL, 0}; - if (load(sender, &module_a) != PURE_OK || load(receiver, &module_b) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } - CHECK(module_a.input[2].prefix == 12); - - /* In memory: a send is its event's; the receiver applies it once per source. */ - PureApp a, b; - CHECK(pure_app_start(&a, &module_a, (PureData){0}) == PURE_OK && pure_app_start(&b, &module_b, (PureData){0}) == PURE_OK); - CHECK(buy(&a, 0, "pen", 2) == PURE_OK && buy(&a, 1, "ink", 3) == PURE_LIMIT); - Outbox o = outbox_of(&a); - CHECK(o.count == 1 && o.items[0].output == 2 && o.items[0].seq == 1 && o.items[0].size == 15 && a.values[0] == 1); - PureSpan sent = {o.items[0].payload, o.items[0].size}; - CHECK(pure_app_deliver(&b, 0, "orders.ordered", 1, sent, NULL) == PURE_OK && !b.duplicate && b.values[0] == 2 && - !strcmp(state_text(&b, 1), "pen")); - pure_app_finish(&b); - CHECK(pure_app_deliver(&b, 0, "orders.ordered", 1, sent, NULL) == PURE_OK && b.duplicate && b.values[0] == 2); - CHECK(deliver(&b, "other", 1, "cup", 5) == PURE_OK && !b.duplicate && b.values[0] == 7); - CHECK(deliver(&b, "orders.ordered", 3, "hat", 1) == PURE_OK && b.values[0] == 8); /* numbers may skip */ - CHECK(deliver(&b, "orders.ordered", 2, "old", 1) == PURE_OK && b.duplicate && b.values[0] == 8); - /* Refused (count 100): delivered all the same, never applied again. */ - CHECK(deliver(&b, "other", 2, "big", 100) == PURE_LIMIT && b.values[0] == 7 + 1); - CHECK(deliver(&b, "other", 2, "big", 1) == PURE_OK && b.duplicate && b.values[0] == 8); - /* Bad deliveries change nothing. */ - CHECK(deliver(&b, "", 4, "x", 1) == PURE_INVALID && deliver(&b, "a b", 4, "x", 1) == PURE_INVALID); - CHECK(deliver(&b, "other", 0, "x", 1) == PURE_INVALID && pure_app_deliver(&a, 2, "s", 1, sent, NULL) == PURE_INVALID); - CHECK(pure_app_message(&a, 2, NULL, -1, sent) == PURE_INVALID); - /* Trimmed, a send is gone at once for the host and from the store at the next save. */ - pure_app_trim(&a, 2, 1); - CHECK(outbox_of(&a).count == 0 && pure_app_save(&a) == PURE_OK && a.sent_count == 0); - CHECK(buy(&a, 0, "cap", 1) == PURE_OK && outbox_of(&a).items[0].seq == 2); - pure_app_stop(&a); - pure_app_stop(&b); - - /* Stored: sends and heard numbers survive restarts; a deferred burst's sends - * appear with its save, and a failed save takes them and their numbers back. */ - char path_a[64], path_b[64]; - snprintf(path_a, sizeof(path_a), "%s", new_store()); - snprintf(path_b, sizeof(path_b), "%s", new_store()); - CHECK(pure_app_start(&a, &module_a, pure_tree_data(path_a)) == PURE_OK); - CHECK(pure_app_start(&b, &module_b, pure_tree_data(path_b)) == PURE_OK); - CHECK(buy(&a, 0, "pen", 2) == PURE_OK && buy(&a, 0, "ink", 3) == PURE_OK); - pure_app_stop(&a); - CHECK(pure_app_start(&a, &module_a, pure_tree_data(path_a)) == PURE_OK); - o = outbox_of(&a); - CHECK(o.count == 2 && o.items[0].seq == 1 && o.items[1].seq == 2 && a.values[0] == 2); - CHECK(pure_app_defer(&a, true)); - for (int i = 0; i < 3; ++i) CHECK(buy(&a, 0, "box", 1) == PURE_OK); - CHECK(outbox_of(&a).count == 2 && pure_app_save(&a) == PURE_OK); - o = outbox_of(&a); - CHECK(o.count == 5 && o.items[4].seq == 5); - CHECK(buy(&a, 0, "lost", 1) == PURE_OK); - pure_app_trim(&a, 2, 3); - fail_stage = PURE_AFTER_HEADER; - CHECK(pure_app_save(&a) == PURE_STORE_ERROR); - fail_stage = -1; - o = outbox_of(&a); - CHECK(o.count == 5 && o.items[0].seq == 1 && a.values[0] == 5); /* the trim and the send undone */ - CHECK(buy(&a, 0, "next", 1) == PURE_OK && pure_app_save(&a) == PURE_OK); - o = outbox_of(&a); - CHECK(o.count == 6 && o.items[5].seq == 6); /* numbered on from 5 */ - pure_app_trim(&a, 2, 4); - CHECK(pure_app_save(&a) == PURE_OK); - pure_app_stop(&a); - CHECK(pure_app_start(&a, &module_a, pure_tree_data(path_a)) == PURE_OK); - o = outbox_of(&a); - CHECK(o.count == 2 && o.items[0].seq == 5 && o.items[1].seq == 6); - /* The receiver: heard numbers stay heard after a restart; a refused one too. */ - CHECK(deliver(&b, "orders.ordered", 1, "pen", 2) == PURE_OK && deliver(&b, "x", 1, "big", 100) == PURE_LIMIT); - pure_app_stop(&b); - CHECK(pure_app_start(&b, &module_b, pure_tree_data(path_b)) == PURE_OK && b.values[0] == 2); - CHECK(deliver(&b, "orders.ordered", 1, "pen", 2) == PURE_OK && b.duplicate && b.values[0] == 2); - CHECK(deliver(&b, "x", 1, "big", 1) == PURE_OK && b.duplicate); - /* A save that fails forgets what it heard. */ - CHECK(pure_app_defer(&b, true) && deliver(&b, "orders.ordered", 2, "ink", 3) == PURE_OK && b.values[0] == 5); - fail_stage = PURE_AFTER_HEADER; - CHECK(pure_app_save(&b) == PURE_STORE_ERROR && b.values[0] == 2); - fail_stage = -1; - CHECK(deliver(&b, "orders.ordered", 2, "ink", 3) == PURE_OK && !b.duplicate && b.values[0] == 5); - CHECK(pure_app_save(&b) == PURE_OK); - pure_app_stop(&b); - - /* Replacing: a module without the output is refused while sends are left - * on it; once they are trimmed it replaces, and the output's number is - * kept for a module that has it again. */ - static const I plain_inputs[] = {{"buy", NULL, PURE_KIND_MESSAGE, false, order_fields, 2}}; - static const O plain_code[] = {{L, 0, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 0, 0}}; - static const H plain_handlers[] = {{1, 0, 4}}; - Spec plain = {sender_nums, 1, plain_inputs, 1, NULL, 0, plain_handlers, 1, plain_code, LEN(plain_code), - NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0}; - CHECK(load(plain, &module_b) == PURE_OK); - CHECK(pure_app_replace(&a, &module_b) == PURE_CONFLICT && a.module == &module_a); - pure_app_trim(&a, 2, 6); - CHECK(pure_app_save(&a) == PURE_OK && pure_app_replace(&a, &module_b) == PURE_OK); - CHECK(buy(&a, 0, "plain", 1) == PURE_OK && pure_app_save(&a) == PURE_OK); - pure_app_stop(&a); - CHECK(pure_app_start(&a, &module_b, pure_tree_data(path_a)) == PURE_OK); /* the number kept in the store */ - CHECK(pure_app_replace(&a, &module_a) == PURE_OK && buy(&a, 0, "back", 1) == PURE_OK && pure_app_save(&a) == PURE_OK); - o = outbox_of(&a); - CHECK(o.count == 1 && o.items[0].seq == 7); - /* A stored send whose output the module lacks refuses the start. */ - pure_app_stop(&a); - CHECK(pure_app_start(&a, &module_b, pure_tree_data(path_a)) == PURE_CONFLICT); - remove_store(path_a); - remove_store(path_b); - - /* A process that dies at each commit stage leaves an event and its send - * together or neither, and a delivery and its effect together or neither. */ - snprintf(path_a, sizeof(path_a), "%s", new_store()); - snprintf(path_b, sizeof(path_b), "%s", new_store()); - CHECK(load(receiver, &module_b) == PURE_OK); - uint64_t heard = 0; - size_t kept[2] = {0, 0}; /* sends and deliveries a death left in place */ - for (int round = 0; round < 2; ++round) - for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { - pid_t child = fork(); - CHECK(child >= 0); - if (!child) { - PureApp doomed, taker; - if (pure_app_start(&doomed, &module_a, pure_tree_data(path_a)) != PURE_OK || - pure_app_start(&taker, &module_b, pure_tree_data(path_b)) != PURE_OK) _exit(1); - crash = 1; - fail_stage = stage; - if (round) buy(&doomed, 0, "crash", 1); - else deliver(&taker, "orders.ordered", heard + 1, "crash", 1); - _exit(0); - } - int status; - CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); - CHECK(pure_app_start(&a, &module_a, pure_tree_data(path_a)) == PURE_OK); - CHECK(pure_app_start(&b, &module_b, pure_tree_data(path_b)) == PURE_OK); - o = outbox_of(&a); - CHECK(o.count == (size_t)a.values[0] && (!o.count || o.items[o.count - 1].seq == o.count)); - heard = b.heard_count ? b.heard[0].seq : 0; - CHECK(b.values[0] == (double)heard); - kept[0] = (size_t)heard; - kept[1] = o.count; - pure_app_stop(&a); - pure_app_stop(&b); - } - CHECK(kept[0] == 1 && kept[1] == 1); /* only a death after the head was written keeps it */ - remove_store(path_a); - remove_store(path_b); - - /* What an output and SEND may not be. */ - const struct { const O *code; size_t count; const char *reason; } refused[] = { - {(const O[]){{FIELD, 1, 0}, {FIELD, 0, 0}, {L, 0, 0}, {SEND, 2, 0}}, 4, "operand type"}, - {(const O[]){{FIELD, 0, 0}, {FIELD, 1, 0}, {L, 0, 0}, {SEND, 1, 0}}, 4, "no such state"}, - {(const O[]){{FIELD, 0, 0}, {FIELD, 1, 0}, {SEND, 2, 0}}, 3, "stack underflow"}, - }; - for (size_t i = 0; i < LEN(refused); ++i) { - Spec bad = sender; - H handler = {1, 0, (uint32_t)refused[i].count}; - bad.o = refused[i].code; bad.no = refused[i].count; bad.h = &handler; bad.nh = 1; - if (load(bad, &module_b) != PURE_INVALID || !strstr(error, refused[i].reason)) { - fprintf(stderr, "links case %zu: expected '%s', got '%s'\n", i, refused[i].reason, error); - abort(); - } - } - H on_output[] = {{1, 0, 8}, {5, 8, 6}}; - Spec bad = sender; - bad.h = on_output; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "handler event")); -} - -/* A timer's tick sees the time the host set; floor and a floored modulo - * take it apart. now is for handlers only. */ -enum { NOW = PURE_OP_NOW, FLOOR = PURE_OP_FLOOR, MOD = PURE_OP_MOD }; -static void timers(void) { - static const S nums[] = {{"clock", 0}, {"seconds", 0}, {"ticks", 0}, {"rest", 0}}; - static const I inputs[] = {TIMER("tick", 1000), KEY("left", "KeyL"), KEY("zero", "KeyZ")}; - static const O code[] = { - {NOW, 0, 0}, {ST, 0, 0}, {NOW, 0, 0}, {C, 0, 1000}, {DIV, 0, 0}, {FLOOR, 0, 0}, {C, 0, 60}, {MOD, 0, 0}, - {ST, 1, 0}, {L, 2, 0}, {C, 0, 1}, {ADD, 0, 0}, {ST, 2, 0}, /* 0 tick */ - {C, 0, -7}, {C, 0, 3}, {MOD, 0, 0}, {C, 0, 7}, {C, 0, -3}, {MOD, 0, 0}, {C, 0, 10}, {MUL, 0, 0}, - {ADD, 0, 0}, {ST, 3, 0}, /* 13 left: 2 + -20 */ - {C, 0, 1}, {C, 0, 0}, {MOD, 0, 0}, {ST, 3, 0}, /* 23 zero */ - }; - static const H handlers[] = {{1, 0, 13}, {3, 13, 10}, {5, 23, 4}}; - static const V one_view[] = {{"v", 0, 1}}; - Spec spec = SPEC(nums, inputs, one_view, handlers, code); - spec.nv = 0; - if (load(spec, &module_a) != PURE_OK) { fprintf(stderr, "%s\n", error); abort(); } - CHECK(module_a.input[0].kind == PURE_KIND_TIMER && module_a.input[0].period == 1000); - PureApp app; - CHECK(pure_app_start(&app, &module_a, (PureData){0}) == PURE_OK); - app.now = 1700000123456.0; - CHECK(pure_app_tick(&app, 0) == PURE_OK && app.values[0] == 1700000123456.0 && - app.values[1] == (double)(1700000123 % 60) && app.values[2] == 1); - CHECK(pure_app_tick(&app, 1) == PURE_INVALID && pure_app_input(&app, 0, PURE_EDGE_DOWN) == PURE_INVALID); - CHECK(pure_app_input(&app, 1, PURE_EDGE_DOWN) == PURE_OK && app.values[3] == 2 - 20); - /* A zero takes the divisor's sign too. */ - static const O zeros[] = {{C, 0, -6}, {C, 0, 3}, {MOD, 0, 0}, {ST, 0, 0}, {C, 0, 6}, {C, 0, -3}, {MOD, 0, 0}, {ST, 1, 0}}; - static const H zero_handler[] = {{3, 0, 8}}; - Spec signs = spec; - signs.o = zeros; signs.no = 8; signs.h = zero_handler; signs.nh = 1; - CHECK(load(signs, &module_b) == PURE_OK); - PureApp zero; - CHECK(pure_app_start(&zero, &module_b, (PureData){0}) == PURE_OK && pure_app_input(&zero, 1, PURE_EDGE_DOWN) == PURE_OK); - CHECK(zero.values[0] == 0 && !signbit(zero.values[0]) && zero.values[1] == 0 && signbit(zero.values[1])); - pure_app_stop(&zero); - CHECK(pure_app_input(&app, 2, PURE_EDGE_DOWN) == PURE_INVALID && app.fault == PURE_FAULT_DIVIDE); - pure_app_stop(&app); - /* A timer's period is 1 ms to a day, and it has nothing else; now is for handlers. */ - static const I no_period[] = {TIMER("tick", 0)}, too_long[] = {TIMER("tick", PURE_TIMER_LONGEST + 1)}; - static const O now_view[] = {{NOW, 0, 0}}; - Spec bad = spec; - bad.in = no_period; bad.ni = 1; bad.nh = 1; bad.o = code; bad.no = 13; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "timer input")); - bad.in = too_long; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "timer input")); - bad = SPEC(nums, inputs, one_view, handlers, now_view); - bad.nh = 0; - CHECK(load(bad, &module_b) == PURE_INVALID && strstr(error, "view with an effect")); -} - -int main(void) { - behaviour(); - faults(); - rejections(); - mutations(); - messages(); - message_fields(); - timers(); - message_mutations(); - texts(); - marks(); - text_mutations(); - lists(); - folds(); - filters(); - keyed(); - keyed_mutations(); - snapshots(); - links(); - fold_mutations(); - pointing(); - list_mutations(); - tables(); - table_mutations(); - stored_states(); - deferred(); - printf("{\"sizeof_module\":%zu,\"sizeof_app\":%zu}\n", sizeof(PureModule), sizeof(PureApp)); - puts("{\"module_tests\":\"verified-load-atomic-events-faults-signed-zero-if-else-replace-by-name-rejections-mutations-messages-documents-typed-verification-text-cursors-utf8-views-rollback-replace-lists-indices-undo-migration-joins-tables-columns-pick-slide-stored-equivalence-crashes-frames-messages-migration-deferred-bursts-pointing-marks-folds-filters-keyed-lists-text-order-message-fields-timers-now-snapshots-links\"}"); - return 0; -} diff --git a/prototypes/connections/test-tree-store.c b/prototypes/connections/test-tree-store.c deleted file mode 100644 index 1130acd..0000000 --- a/prototypes/connections/test-tree-store.c +++ /dev/null @@ -1,1131 +0,0 @@ -#define _POSIX_C_SOURCE 200809L -#include "store.h" -#include "state.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define CHECK(x) do { if (!(x)) { fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #x); abort(); } } while (0) - -static int fail_stage = -1; -static int crash; -static int slow_once; /* the next publication pauses before its data sync */ -static int held_sync; /* 1: the next publication waits after its head write; 2: it is waiting */ -static void pause_ms(long ms) { - struct timespec wait = {ms / 1000, (ms % 1000) * 1000000}; - nanosleep(&wait, NULL); -} -int pure_store_test_fault(PureStoreStage stage) { - if (stage == PURE_BEFORE_FILE_SYNC && __atomic_exchange_n(&slow_once, 0, __ATOMIC_SEQ_CST)) pause_ms(100); - int one = 1; - if (stage == PURE_BEFORE_DIR_SYNC && - __atomic_compare_exchange_n(&held_sync, &one, 2, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) - while (__atomic_load_n(&held_sync, __ATOMIC_SEQ_CST) == 2) pause_ms(1); - if ((int)stage != fail_stage) return 0; - if (crash) _exit(77); - return 1; -} -bool pure_tree_test_shape(const char *path, const char *key, unsigned *height, - uint64_t *leaves, uint64_t *small, uint64_t *file_end); - -void pure_draft_test_arena(const PureDraft *draft, size_t *nodes, size_t *bytes); -unsigned long pure_tree_test_publications(void); -unsigned long pure_tree_test_syncs(void); -void pure_tree_test_forget(void); - -/* The file: head slots in blocks 0 and 1 (sequence at +8, end at +16, - * catalog at +24), commits from DATA_START, each a 128-byte record (its head) - * and then its bytes. */ -enum { KEYS = 4, CAPACITY = 1 << 20, SLOT_BLOCK = 4096, DATA_START = 8192, RECORD_SIZE = 128 }; -typedef struct { unsigned char *bytes; size_t size; int64_t revision; } Oracle; - -static uint32_t random_value(uint32_t *state) { - *state ^= *state << 13; - *state ^= *state >> 17; - *state ^= *state << 5; - return *state; -} - -static void expect(const char *path, const char *key, const Oracle *o) { - PureStoredView view; - CHECK(pure_store_get(path, key, PURE_OBJECT_LIMIT, &view) == PURE_OK); - CHECK(view.revision == o->revision && view.body.size == o->size); - CHECK(!o->size || !memcmp(view.body.data, o->bytes, o->size)); - pure_store_release(&view); -} - -static uint64_t file_size(const char *path) { - char name[512]; - snprintf(name, sizeof(name), "%s/pure.tree", path); - struct stat info; - CHECK(stat(name, &info) == 0); - return (uint64_t)info.st_size; -} - -/* Where the newest commit ends. */ -static uint64_t published_end(const char *path, const char *key) { - unsigned height; - uint64_t leaves, small, end; - CHECK(pure_tree_test_shape(path, key, &height, &leaves, &small, &end)); - return end; -} - -static uint64_t field_at(int fd, uint64_t at) { - unsigned char bytes[8]; - CHECK(pread(fd, bytes, 8, (off_t)at) == 8); - uint64_t value = 0; - for (unsigned i = 0; i < 8; ++i) value |= (uint64_t)bytes[i] << (8 * i); - return value; -} - -static uint64_t slot_field(int fd, unsigned slot, unsigned offset) { return field_at(fd, slot * SLOT_BLOCK + offset); } - -/* Where the newest head is: the newer slot's, or the last record after it. */ -static uint64_t newest_head(int fd) { - uint64_t at = slot_field(fd, 0, 8) > slot_field(fd, 1, 8) ? 0 : SLOT_BLOCK; - for (;;) { - uint64_t end = field_at(fd, at + 16); - unsigned char magic[8]; - if (pread(fd, magic, 8, (off_t)end) != 8 || memcmp(magic, "PUREREC1", 8) || - field_at(fd, end + 8) != field_at(fd, at + 8) + 1) return at; - at = end; - } -} - -static void fill(unsigned char *bytes, size_t size, uint32_t *state) { - for (size_t i = 0; i < size; ++i) bytes[i] = (unsigned char)random_value(state); -} - -/* Random edits against a flat oracle, including views held across later - * commits and compactions. Every stored node must stay balanced. */ -static void oracle(const char *path) { - static const char *keys[KEYS] = {"alpha", "b", "gamma-long_key-0123456789", "z"}; - Oracle docs[KEYS] = {{0}}; - unsigned char *insert = malloc(65536); - for (unsigned k = 0; k < KEYS; ++k) CHECK((docs[k].bytes = malloc(CAPACITY))); - CHECK(insert); - uint32_t state = 0x9e3779b9u; - PureStoredView held = {0}; - unsigned char *held_copy = malloc(CAPACITY); - size_t held_size = 0; - unsigned max_height = 0; - size_t max_size = 0, splices = 0; - for (unsigned step = 0; step < 6000; ++step) { - unsigned k = random_value(&state) % KEYS; - Oracle *o = &docs[k]; - uint32_t kind = random_value(&state) % 100; - size_t count = kind < 70 ? random_value(&state) % 24 : - kind < 95 ? random_value(&state) % 6000 : random_value(&state) % 65536; - int64_t revision; - fill(insert, count, &state); - if (kind >= 98 || !o->revision) { - CHECK(pure_store_put(path, keys[k], o->revision, (PureSpan){insert, count}, &revision) == PURE_OK); - memcpy(o->bytes, insert, count); - o->size = count; - } else { - size_t offset = o->size ? random_value(&state) % (o->size + 1) : 0; - size_t remove = o->size - offset ? random_value(&state) % (o->size - offset + 1) : 0; - if (random_value(&state) % 24) remove = remove % 64; - if (o->size - remove + count > CAPACITY) { count = 0; remove = o->size - offset; } - CHECK(pure_store_splice(path, keys[k], o->revision, offset, remove, - (PureSpan){insert, count}, PURE_OBJECT_LIMIT, &revision) == PURE_OK); - memmove(o->bytes + offset + count, o->bytes + offset + remove, o->size - offset - remove); - memcpy(o->bytes + offset, insert, count); - o->size = o->size - remove + count; - ++splices; - } - if (o->size > max_size) max_size = o->size; - CHECK(revision == o->revision + 1); - o->revision = revision; - expect(path, keys[k], o); - unsigned height; - uint64_t leaves, small, end; - CHECK(pure_tree_test_shape(path, keys[k], &height, &leaves, &small, &end)); - CHECK(small == 0); - if (height > max_height) max_height = height; - if (step % 97 == 0) { - if (held.mapped_size) { - CHECK(held.body.size == held_size && !memcmp(held.body.data, held_copy, held_size)); - pure_store_release(&held); - } - CHECK(pure_store_get(path, keys[k], PURE_OBJECT_LIMIT, &held) == PURE_OK); - held_size = held.body.size; - if (held_size) memcpy(held_copy, held.body.data, held_size); - } - } - if (held.mapped_size) { - CHECK(held.body.size == held_size && !memcmp(held.body.data, held_copy, held_size)); - pure_store_release(&held); - } - for (unsigned k = 0; k < KEYS; ++k) { expect(path, keys[k], &docs[k]); free(docs[k].bytes); } - CHECK(max_height >= 2 && max_height <= 4 && max_size > 262144); - printf("{\"oracle_steps\":6000,\"splices\":%zu,\"max_document_bytes\":%zu,\"max_height\":%u}\n", - splices, max_size, max_height); - free(insert); - free(held_copy); -} - -/* A one-byte insert, delete or replacement in a 1 MiB document appends only - * its changed leaf and path, not another copy of the document. */ -static void amplification(const char *path) { - size_t size = 1048576; - unsigned char *bytes = malloc(size + 1); - uint32_t state = 7; - CHECK(bytes); - fill(bytes, size, &state); - int64_t revision; - CHECK(pure_store_put(path, "large", 0, (PureSpan){bytes, size}, &revision) == PURE_OK); - Oracle o = {bytes, size, revision}; - unsigned char byte = 'Q'; - const size_t edits[][3] = {{size / 2, 0, 1}, {size / 3, 1, 0}, {size - 1, 1, 1}, {0, 0, 1}}; - uint64_t worst = 0; - for (size_t i = 0; i < sizeof(edits) / sizeof(*edits); ++i) { - uint64_t before = published_end(path, "large"); - size_t offset = edits[i][0], remove = edits[i][1], count = edits[i][2]; - CHECK(pure_store_splice(path, "large", o.revision, offset, remove, (PureSpan){&byte, count}, - PURE_OBJECT_LIMIT, &revision) == PURE_OK); - memmove(bytes + offset + count, bytes + offset + remove, o.size - offset - remove); - memcpy(bytes + offset, &byte, count); - o.size = o.size - remove + count; - o.revision = revision; - uint64_t after = published_end(path, "large"); - if (after > before && after - before > worst) worst = after - before; - expect(path, "large", &o); - } - /* Compaction may shrink the file; growth per edit is the claim. */ - CHECK(worst > 0 && worst < 16384); - unsigned height; - uint64_t leaves, small, end; - CHECK(pure_tree_test_shape(path, "large", &height, &leaves, &small, &end)); - CHECK(small == 0 && height == 2 && leaves >= 256 && leaves <= 512); - printf("{\"one_byte_edit_max_appended_bytes\":%llu,\"height\":%u,\"leaves\":%llu}\n", - (unsigned long long)worst, height, (unsigned long long)leaves); - free(bytes); -} - -static void faults(const char *path) { - unsigned char text[] = "fault-free"; - int64_t revision; - CHECK(pure_store_put(path, "fault", 0, (PureSpan){text, 10}, &revision) == PURE_OK); - Oracle o = {text, 10, revision}; - uint64_t size = file_size(path), end = published_end(path, "fault"); - for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; ++fail_stage) { - CHECK(pure_store_splice(path, "fault", o.revision, 1, 3, (PureSpan){text, 5}, - PURE_OBJECT_LIMIT, &revision) == PURE_STORE_ERROR); - expect(path, "fault", &o); - /* The bytes stayed unpublished, in the written zeros after the end. */ - CHECK(published_end(path, "fault") == end && file_size(path) == size); - } - fail_stage = PURE_BEFORE_DIR_SYNC; - CHECK(pure_store_splice(path, "fault", o.revision, 10, 0, (PureSpan){text, 2}, - PURE_OBJECT_LIMIT, &revision) == PURE_UNCERTAIN); - fail_stage = -1; - unsigned char next[] = "fault-freefa"; - o = (Oracle){next, 12, o.revision + 1}; - CHECK(revision == o.revision); - expect(path, "fault", &o); -} - -/* A process that dies at each publication stage leaves either the previous - * or the next complete version; the next commit overwrites its unpublished - * bytes. A head it wrote but did not sync is made durable before the next - * writer builds on it. */ -static void crashes(const char *path) { - unsigned char text[] = "crash-base"; - int64_t revision; - CHECK(pure_store_put(path, "crash", 0, (PureSpan){text, 10}, &revision) == PURE_OK); - Oracle o = {text, 10, revision}; - for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { - uint64_t before = published_end(path, "crash"); - pid_t child = fork(); - CHECK(child >= 0); - if (!child) { - crash = 1; - fail_stage = stage; - pure_store_splice(path, "crash", o.revision, 0, 1, (PureSpan){(const unsigned char *)"C", 1}, - PURE_OBJECT_LIMIT, &revision); - _exit(0); - } - int status; - CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); - if (stage == PURE_BEFORE_DIR_SYNC) { text[0] = 'C'; ++o.revision; } - expect(path, "crash", &o); - uint64_t published = published_end(path, "crash"); - CHECK(stage == PURE_BEFORE_DIR_SYNC ? published > before : published == before); - unsigned long syncs = pure_tree_test_syncs(); - CHECK(pure_store_splice(path, "crash", o.revision, 10, 0, (PureSpan){(const unsigned char *)"", 0}, - PURE_OBJECT_LIMIT, &revision) == PURE_OK); - o.revision = revision; - expect(path, "crash", &o); - CHECK(pure_tree_test_syncs() - syncs == (stage == PURE_BEFORE_DIR_SYNC ? 2u : 1u)); - CHECK(published_end(path, "crash") > published); - } -} - -static void tear(const char *path, uint64_t at) { - char name[512]; - snprintf(name, sizeof(name), "%s/pure.tree", path); - int fd = open(name, O_RDWR); - unsigned char torn; - CHECK(fd >= 0 && pread(fd, &torn, 1, (off_t)at) == 1); - torn ^= 0x5a; - CHECK(pwrite(fd, &torn, 1, (off_t)at) == 1 && close(fd) == 0); -} - -/* A torn newest record exposes the previous commit to a new process, and the - * next writer overwrites that record and the commit's bytes. A torn slot - * costs only a longer walk: the other slot's head leads along the records. */ -static void torn_head(const char *path) { - unsigned char first[] = "first", second[] = "second"; - int64_t revision; - CHECK(pure_store_put(path, "torn", 0, (PureSpan){first, 5}, &revision) == PURE_OK); - Oracle o = {first, 5, revision}; - uint64_t record = published_end(path, "torn"); /* where the next commit's record goes */ - CHECK(pure_store_put(path, "torn", o.revision, (PureSpan){second, 6}, &revision) == PURE_OK); - tear(path, SLOT_BLOCK + 20); - pure_tree_test_forget(); - expect(path, "torn", &(Oracle){second, 6, revision}); - tear(path, record + 20); - pure_tree_test_forget(); - expect(path, "torn", &o); - CHECK(pure_store_put(path, "torn", o.revision + 1, (PureSpan){first, 1}, &revision) == PURE_CONFLICT); - CHECK(pure_store_put(path, "torn", o.revision, (PureSpan){second, 3}, &revision) == PURE_OK); - o = (Oracle){second, 3, revision}; - CHECK(published_end(path, "torn") < record + RECORD_SIZE + 4096); /* it went where the torn one was */ - pure_tree_test_forget(); - expect(path, "torn", &o); -} - -/* Commits are found along their records: from a slot's head, which every - * CHECKPOINT commits moves on, by a new process too; a damaged record ends - * the walk at the commit before it, which the next writer builds on. */ -static void records(const char *path) { - int64_t revision; - uint64_t ends[41]; - char text[16]; - CHECK(pure_store_put(path, "walk", 0, (PureSpan){(const unsigned char *)"0", 1}, &revision) == PURE_OK); - for (unsigned i = 1; i <= 40; ++i) { - ends[i] = published_end(path, "walk"); - snprintf(text, sizeof(text), "%u", i); - CHECK(pure_store_put(path, "walk", revision, (PureSpan){(const unsigned char *)text, strlen(text)}, - &revision) == PURE_OK); - } - char name[512]; - snprintf(name, sizeof(name), "%s/pure.tree", path); - int fd = open(name, O_RDONLY); - CHECK(fd >= 0); - uint64_t newest_slot = slot_field(fd, 0, 8) > slot_field(fd, 1, 8) ? slot_field(fd, 0, 8) : slot_field(fd, 1, 8); - CHECK(close(fd) == 0); - /* 42 commits (the empty file's first): heads in slots at 17 and 33. */ - CHECK(newest_slot == 33); - pure_tree_test_forget(); - expect(path, "walk", &(Oracle){(unsigned char *)"40", 2, 41}); - pid_t child = fork(); - CHECK(child >= 0); - if (!child) { - pure_tree_test_forget(); /* a process that never saw them */ - PureStoredView view; - _exit(pure_store_get(path, "walk", 16, &view) == PURE_OK && view.body.size == 2 && - !memcmp(view.body.data, "40", 2) && view.revision == 41 ? 0 : 1); - } - int status; - CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 0); - /* Commit 37 ("37", revision 38) has a damaged record: the walk ends at 36. */ - tear(path, ends[37] + 30); - pure_tree_test_forget(); - expect(path, "walk", &(Oracle){(unsigned char *)"36", 2, 37}); - CHECK(pure_store_put(path, "walk", 37, (PureSpan){(const unsigned char *)"x", 1}, &revision) == PURE_OK); - CHECK(revision == 38 && published_end(path, "walk") < ends[38]); - pure_tree_test_forget(); - expect(path, "walk", &(Oracle){(unsigned char *)"x", 1, 38}); -} - -/* A reader's view stays byte-identical while another process commits and - * compacts; readers take no lock. */ -static void isolation(const char *path) { - size_t size = 300000; - unsigned char *bytes = malloc(size); - uint32_t state = 99; - CHECK(bytes); - fill(bytes, size, &state); - int64_t revision; - CHECK(pure_store_put(path, "shared", 0, (PureSpan){bytes, size}, &revision) == PURE_OK); - PureStoredView view; - CHECK(pure_store_get(path, "shared", size, &view) == PURE_OK); - uint64_t before = file_size(path); - pid_t child = fork(); - CHECK(child >= 0); - if (!child) { - int64_t at = revision; - unsigned char byte = 'x'; - for (unsigned i = 0; i < 400; ++i) - if (pure_store_splice(path, "shared", at, (i * 7919u) % size, 1, (PureSpan){&byte, 1}, - PURE_OBJECT_LIMIT, &at) != PURE_OK) _exit(1); - _exit(0); - } - int status; - CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 0); - CHECK(view.revision == revision && view.body.size == size && !memcmp(view.body.data, bytes, size)); - pure_store_release(&view); - PureStoredView latest; - CHECK(pure_store_get(path, "shared", size, &latest) == PURE_OK && latest.revision == revision + 400); - pure_store_release(&latest); - /* About 2 MB of path copies without compaction would exceed this. */ - CHECK(file_size(path) < before + 3 * 524288); - free(bytes); -} - -/* Damaged branch bytes are reported as storage errors, never trusted. */ -static void corruption(const char *path) { - unsigned char *bytes = malloc(200000); - uint32_t state = 3; - CHECK(bytes); - fill(bytes, 200000, &state); - int64_t revision; - CHECK(pure_store_put(path, "damaged", 0, (PureSpan){bytes, 200000}, &revision) == PURE_OK); - char name[512]; - snprintf(name, sizeof(name), "%s/pure.tree", path); - uint64_t end = published_end(path, "damaged"); - int fd = open(name, O_RDWR); - CHECK(fd >= 0); - unsigned errors = 0; - for (unsigned i = 0; i < 3000; ++i) { - off_t at = (off_t)(DATA_START + random_value(&state) % (end - DATA_START)); - unsigned char original, damaged; - CHECK(pread(fd, &original, 1, at) == 1); - damaged = original ^ (unsigned char)(1u << (random_value(&state) % 8)); - CHECK(pwrite(fd, &damaged, 1, at) == 1); - PureStoredView view; - PureResult result = pure_store_get(path, "damaged", PURE_OBJECT_LIMIT, &view); - CHECK(result == PURE_OK || result == PURE_STORE_ERROR || result == PURE_NOT_FOUND); - if (result == PURE_OK) pure_store_release(&view); else ++errors; - CHECK(pwrite(fd, &original, 1, at) == 1); - } - CHECK(close(fd) == 0 && errors > 0); - free(bytes); -} - -static void many_keys(const char *path) { - int64_t revision; - char key[32]; - for (unsigned i = 0; i < 1024; ++i) { - unsigned id = (i * 617u) % 1024u; - snprintf(key, sizeof(key), "doc-%04u", id); - CHECK(pure_store_put(path, key, 0, (PureSpan){(const unsigned char *)key, strlen(key)}, &revision) == PURE_OK); - } - for (unsigned id = 0; id < 1024; ++id) { - snprintf(key, sizeof(key), "doc-%04u", id); - Oracle o = {(unsigned char *)key, strlen(key), 1}; - expect(path, key, &o); - } - PureStoredView view; - CHECK(pure_store_get(path, "doc-1024", 64, &view) == PURE_NOT_FOUND); - CHECK(pure_store_get(path, "doc-0001", 3, &view) == PURE_BUDGET); - CHECK(pure_store_splice(path, "doc-0002", 1, 9, 0, (PureSpan){0}, PURE_OBJECT_LIMIT, &revision) == PURE_LIMIT); - CHECK(pure_store_put(path, "doc-0003", 0, (PureSpan){0}, &revision) == PURE_CONFLICT && revision == 1); -} - -/* The newest catalog of a one-key store is one leaf; forge its revision. */ -static void revision_limit(const char *path) { - int64_t revision; - CHECK(pure_store_put(path, "work", 0, (PureSpan){(const unsigned char *)"w", 1}, &revision) == PURE_OK); - char name[512]; - snprintf(name, sizeof(name), "%s/pure.tree", path); - int fd = open(name, O_RDWR); - CHECK(fd >= 0); - uint64_t catalog = field_at(fd, newest_head(fd) + 24); - CHECK(catalog >> 63); - const unsigned char maximum[] = {255,255,255,255,255,255,255,127}; - off_t record = (off_t)(catalog & ((UINT64_C(1) << 62) - 1)); - CHECK(pwrite(fd, maximum, 8, record + 80) == 8 && close(fd) == 0); - /* The forged commit no longer matches its digest; this process wrote it - * and does not read it again, a new process would pass over it. */ - PureStoredView view; - CHECK(pure_store_get(path, "work", 1, &view) == PURE_OK && view.revision == INT64_MAX); - pure_store_release(&view); - CHECK(pure_store_splice(path, "work", INT64_MAX, 0, 0, (PureSpan){0}, PURE_OBJECT_LIMIT, &revision) == PURE_LIMIT); -} - -static bool count_key(void *context, const char *key) { - (void)key; - ++*(unsigned *)context; - return true; -} - -static void keys(const char *path) { - unsigned count = 0; - CHECK(pure_store_keys(path, count_key, &count) == PURE_OK && count == 0); - int64_t revision; - const char *names[] = {"m", "a", "zz", "a-b", "A"}; - for (unsigned i = 0; i < 5; ++i) - CHECK(pure_store_put(path, names[i], 0, (PureSpan){0}, &revision) == PURE_OK); - CHECK(pure_store_keys(path, count_key, &count) == PURE_OK && count == 5); -} - -typedef struct { unsigned char *out; size_t spans, largest, limit; } Gather; - -static bool gather(void *context, PureSpan span) { - Gather *g = context; - memcpy(g->out, span.data, span.size); - g->out += span.size; - if (span.size > g->largest) g->largest = span.size; - return ++g->spans != g->limit; -} - -static void expect_draft(const PureDraft *draft, const Oracle *o, unsigned char *scratch) { - Gather g = {scratch, 0, 0, 0}; - CHECK(pure_draft_size(draft) == o->size && pure_draft_revision(draft) == o->revision); - CHECK(pure_draft_read(draft, 0, o->size, gather, &g) == PURE_OK); - CHECK((size_t)(g.out - scratch) == o->size && (!o->size || !memcmp(scratch, o->bytes, o->size))); - CHECK(g.largest <= 4096); -} - -static void apply(Oracle *o, size_t offset, size_t remove, const unsigned char *insert, size_t count) { - memmove(o->bytes + offset + count, o->bytes + offset + remove, o->size - offset - remove); - if (count) memcpy(o->bytes + offset, insert, count); - o->size = o->size - remove + count; -} - -/* Edits live only in the draft until commit; each commit continues from the - * published version with an empty arena. */ -static void drafts(const char *path) { - Oracle o = {malloc(CAPACITY), 0, 0}; - unsigned char *scratch = malloc(CAPACITY), *insert = malloc(65536); - CHECK(o.bytes && scratch && insert); - PureDraft *draft; - PureStoredView view; - CHECK(pure_draft_open(path, "draft", &draft) == PURE_OK); - expect_draft(draft, &o, scratch); - uint32_t state = 0x1234567u; - for (unsigned step = 0; step < 20000; ++step) { - uint32_t kind = random_value(&state) % 100; - size_t count = kind < 80 ? random_value(&state) % 8 : - kind < 98 ? random_value(&state) % 4000 : random_value(&state) % 65536; - size_t offset = o.size ? random_value(&state) % (o.size + 1) : 0; - size_t remove = o.size - offset ? random_value(&state) % (o.size - offset + 1) : 0; - if (random_value(&state) % 64) remove %= 16; - if (o.size - remove + count > CAPACITY) count = 0; - fill(insert, count, &state); - CHECK(pure_draft_splice(draft, offset, remove, (PureSpan){insert, count}) == PURE_OK); - apply(&o, offset, remove, insert, count); - if (step % 97 == 0) expect_draft(draft, &o, scratch); - if (step % 1000 == 999) { - if (o.revision) CHECK(pure_store_get(path, "draft", PURE_OBJECT_LIMIT, &view) == PURE_OK && - view.revision == o.revision); - if (o.revision) pure_store_release(&view); - int64_t revision; - CHECK(pure_draft_commit(draft, &revision) == PURE_OK && revision == o.revision + 1); - o.revision = revision; - expect(path, "draft", &o); - expect_draft(draft, &o, scratch); - size_t nodes, bytes; - pure_draft_test_arena(draft, &nodes, &bytes); - CHECK(nodes == 0); - } - } - CHECK(o.size > 100000); - Gather g = {scratch, 0, 0, 3}; - CHECK(pure_draft_read(draft, 0, o.size, gather, &g) == PURE_OK && g.spans == 3); - CHECK(pure_draft_splice(draft, o.size + 1, 0, (PureSpan){0}) == PURE_LIMIT); - CHECK(pure_draft_splice(draft, 0, o.size + 1, (PureSpan){0}) == PURE_LIMIT); - CHECK(pure_draft_read(draft, o.size, 1, gather, &g) == PURE_LIMIT); - expect_draft(draft, &o, scratch); - pure_draft_close(draft); - printf("{\"draft_steps\":20000,\"draft_commits\":%lld,\"draft_bytes\":%zu}\n", - (long long)o.revision, o.size); - free(o.bytes); - free(scratch); - free(insert); -} - -/* Long typing and deleting at one place keeps only the live path in memory, - * and its commit appends the typed bytes and path, not the document. */ -static void typing(const char *path) { - size_t size = 1048576; - Oracle o = {malloc(size + 40000), size, 0}; - unsigned char *scratch = malloc(size + 40000); - uint32_t state = 5; - CHECK(o.bytes && scratch); - fill(o.bytes, size, &state); - CHECK(pure_store_put(path, "typed", 0, (PureSpan){o.bytes, size}, &o.revision) == PURE_OK); - PureDraft *draft; - CHECK(pure_draft_open(path, "typed", &draft) == PURE_OK); - size_t cursor = size / 2, nodes, bytes, peak = 0; - for (unsigned i = 0; i < 30000; ++i) { - unsigned char byte = (unsigned char)('a' + i % 26); - if (i % 3 == 2) { - CHECK(pure_draft_splice(draft, cursor - 1, 1, (PureSpan){0}) == PURE_OK); - apply(&o, --cursor, 1, NULL, 0); - } else { - CHECK(pure_draft_splice(draft, cursor, 0, (PureSpan){&byte, 1}) == PURE_OK); - apply(&o, cursor++, 0, &byte, 1); - } - pure_draft_test_arena(draft, &nodes, &bytes); - if (bytes > peak) peak = bytes; - } - expect_draft(draft, &o, scratch); - /* Reachable: 10,000 net typed bytes, the leaves they landed in and one - * path. Unreachable copies are reclaimed past max(reachable, 32 KiB). */ - CHECK(peak < 2 * (10000 + 4 * 4096 + 2048) + 32768 + 8192); - uint64_t before = published_end(path, "typed"); - int64_t revision; - CHECK(pure_draft_commit(draft, &revision) == PURE_OK); - o.revision = revision; - uint64_t appended = published_end(path, "typed") - before; - CHECK(appended < 10000 + 16384); - expect(path, "typed", &o); - pure_draft_close(draft); - printf("{\"typing_keystrokes\":30000,\"peak_arena_bytes\":%zu,\"commit_appended_bytes\":%llu}\n", - peak, (unsigned long long)appended); - free(o.bytes); - free(scratch); -} - -/* Two drafts of one key: the second commit conflicts and keeps its edits. */ -static void draft_conflict(const char *path) { - unsigned char scratch[64]; - int64_t revision; - CHECK(pure_store_put(path, "shared", 0, (PureSpan){(const unsigned char *)"base", 4}, &revision) == PURE_OK); - PureDraft *first, *second; - CHECK(pure_draft_open(path, "shared", &first) == PURE_OK && pure_draft_open(path, "shared", &second) == PURE_OK); - CHECK(pure_draft_splice(first, 4, 0, (PureSpan){(const unsigned char *)"-one", 4}) == PURE_OK); - CHECK(pure_draft_splice(second, 0, 0, (PureSpan){(const unsigned char *)"two-", 4}) == PURE_OK); - CHECK(pure_draft_commit(first, &revision) == PURE_OK && revision == 2); - CHECK(pure_draft_commit(second, &revision) == PURE_CONFLICT && revision == 2); - Oracle kept = {(unsigned char *)"two-base", 8, 1}; - expect_draft(second, &kept, scratch); - Oracle stored = {(unsigned char *)"base-one", 8, 2}; - expect(path, "shared", &stored); - pure_draft_close(second); - CHECK(pure_draft_open(path, "shared", &second) == PURE_OK); - expect_draft(second, &stored, scratch); - CHECK(pure_store_put(path, "shared", 2, (PureSpan){(const unsigned char *)"x", 1}, &revision) == PURE_OK); - CHECK(pure_draft_commit(first, &revision) == PURE_CONFLICT && revision == 3); - pure_draft_close(first); - pure_draft_close(second); -} - -/* Another key's writes compact the file under an open draft; the draft's - * stored extents are copied from its old mapping on commit. */ -static void draft_relocation(const char *path) { - unsigned char *big = malloc(300000), scratch[20000]; - uint32_t state = 11; - CHECK(big); - fill(big, 300000, &state); - Oracle o = {malloc(20000), 12000, 0}; - CHECK(o.bytes); - fill(o.bytes, o.size, &state); - CHECK(pure_store_put(path, "kept", 0, (PureSpan){o.bytes, o.size}, &o.revision) == PURE_OK); - PureDraft *draft; - CHECK(pure_draft_open(path, "kept", &draft) == PURE_OK); - CHECK(pure_draft_splice(draft, 6000, 10, (PureSpan){(const unsigned char *)"relocated", 9}) == PURE_OK); - apply(&o, 6000, 10, (const unsigned char *)"relocated", 9); - char name[512]; - snprintf(name, sizeof(name), "%s/pure.tree", path); - struct stat before, after; - CHECK(stat(name, &before) == 0); - int64_t revision = 0; - for (unsigned i = 0; i < 8; ++i) - CHECK(pure_store_put(path, "other", revision, (PureSpan){big, 300000 - i}, &revision) == PURE_OK); - CHECK(stat(name, &after) == 0 && after.st_ino != before.st_ino); - CHECK(pure_draft_commit(draft, &revision) == PURE_OK && revision == o.revision + 1); - o.revision = revision; - expect(path, "kept", &o); - expect_draft(draft, &o, scratch); - Oracle other = {big, 300000 - 7, 8}; - expect(path, "other", &other); - pure_draft_close(draft); - free(o.bytes); - free(big); -} - -/* A held draft keeps earlier versions readable across later edits. */ -static void draft_versions(const char *path) { - static unsigned char bytes[200000], scratch[200000]; - uint32_t state = 17; - fill(bytes, sizeof(bytes), &state); - int64_t revision; - CHECK(pure_store_put(path, "versioned", 0, (PureSpan){bytes, sizeof(bytes)}, &revision) == PURE_OK); - PureDraft *draft; - CHECK(pure_draft_open(path, "versioned", &draft) == PURE_OK); - pure_draft_hold(draft, true); - PureVersion first = pure_draft_version(draft); - for (unsigned i = 0; i < 400; ++i) { - unsigned char byte = (unsigned char)i; - CHECK(pure_draft_splice(draft, (i * 977u) % 150000, 3, (PureSpan){&byte, 1}) == PURE_OK); - } - Gather g = {scratch, 0, 0, 0}; - CHECK(pure_draft_read_version(draft, first, 0, sizeof(bytes), gather, &g) == PURE_OK); - CHECK((size_t)(g.out - scratch) == sizeof(bytes) && !memcmp(scratch, bytes, sizeof(bytes))); - CHECK(pure_draft_size(draft) == sizeof(bytes) - 800); - size_t nodes, held_bytes, after; - pure_draft_test_arena(draft, &nodes, &held_bytes); - pure_draft_hold(draft, false); - pure_draft_test_arena(draft, &nodes, &after); - CHECK(after < held_bytes); - pure_draft_close(draft); -} - -/* A draft without a store: edits, versions, restore and reclamation. */ -static void anonymous_draft(const char *path) { - (void)path; - Oracle o = {malloc(CAPACITY), 0, 0}; - unsigned char *scratch = malloc(CAPACITY), insert[3000]; - PureDraft *draft = pure_draft_new(); - CHECK(draft && o.bytes && scratch && pure_draft_size(draft) == 0 && pure_draft_revision(draft) == 0); - uint32_t state = 42; - for (unsigned step = 0; step < 4000; ++step) { - size_t count = random_value(&state) % (step % 50 ? 12 : sizeof(insert)); - size_t offset = o.size ? random_value(&state) % (o.size + 1) : 0; - size_t remove = o.size - offset ? random_value(&state) % (o.size - offset + 1) % 32 : 0; - fill(insert, count, &state); - bool abandon = random_value(&state) % 5 == 0; - pure_draft_hold(draft, true); - PureVersion before = pure_draft_version(draft); - CHECK(pure_draft_splice(draft, offset, remove, (PureSpan){insert, count}) == PURE_OK); - if (abandon) pure_draft_restore(draft, before); - else apply(&o, offset, remove, insert, count); - pure_draft_hold(draft, false); - if (step % 37 == 0) expect_draft(draft, &o, scratch); - } - expect_draft(draft, &o, scratch); - int64_t revision; - CHECK(pure_draft_commit(draft, &revision) == PURE_INVALID); - size_t nodes, bytes; - pure_draft_test_arena(draft, &nodes, &bytes); - CHECK(bytes < 2 * o.size + 65536); - pure_draft_close(draft); - free(o.bytes); - free(scratch); -} - -/* A failed commit changes neither the store nor the draft. */ -static void draft_faults(const char *path) { - unsigned char scratch[64]; - int64_t revision; - CHECK(pure_store_put(path, "faulty", 0, (PureSpan){(const unsigned char *)"stable", 6}, &revision) == PURE_OK); - PureDraft *draft; - CHECK(pure_draft_open(path, "faulty", &draft) == PURE_OK); - CHECK(pure_draft_splice(draft, 6, 0, (PureSpan){(const unsigned char *)"+edit", 5}) == PURE_OK); - Oracle stored = {(unsigned char *)"stable", 6, 1}, edited = {(unsigned char *)"stable+edit", 11, 1}; - uint64_t size = file_size(path); - for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; ++fail_stage) { - CHECK(pure_draft_commit(draft, &revision) == PURE_STORE_ERROR); - expect(path, "faulty", &stored); - expect_draft(draft, &edited, scratch); - CHECK(file_size(path) == size); - } - fail_stage = PURE_BEFORE_DIR_SYNC; - CHECK(pure_draft_commit(draft, &revision) == PURE_UNCERTAIN && revision == 2); - fail_stage = -1; - expect_draft(draft, &edited, scratch); - CHECK(pure_draft_commit(draft, &revision) == PURE_CONFLICT && revision == 2); - pure_draft_close(draft); - CHECK(pure_draft_open(path, "faulty", &draft) == PURE_OK); - edited.revision = 2; - expect_draft(draft, &edited, scratch); - CHECK(pure_draft_splice(draft, 0, 6, (PureSpan){0}) == PURE_OK); - CHECK(pure_draft_commit(draft, &revision) == PURE_OK && revision == 3); - Oracle final = {(unsigned char *)"+edit", 5, 3}; - expect(path, "faulty", &final); - pure_draft_close(draft); - CHECK(pure_draft_open(path, "bad/key", &draft) == PURE_INVALID && !draft); -} - -static bool list_key(void *context, const char *key) { - char *seen = context; - strcat(seen, key); - strcat(seen, ","); - return true; -} - -/* Several drafts publish in one commit: all or none, under process crashes, - * faults and a stale draft. Runtime values stay out of the document keys. */ -static void batch(const char *path) { - unsigned char scratch[64]; - int64_t revision, revisions[3]; - CHECK(pure_store_put(path, "alpha", 0, (PureSpan){(const unsigned char *)"a0", 2}, &revision) == PURE_OK); - CHECK(pure_store_put(path, "beta", 0, (PureSpan){(const unsigned char *)"b0", 2}, &revision) == PURE_OK); - PureDraft *drafts[3]; - CHECK(pure_draft_open(path, "alpha", &drafts[0]) == PURE_OK && pure_draft_open(path, "beta", &drafts[1]) == PURE_OK); - CHECK(pure_state_open(path, "values", &drafts[2]) == PURE_OK && pure_draft_revision(drafts[2]) == 0); - PureDraft *bad; - CHECK(pure_state_open(path, "bad.name", &bad) == PURE_INVALID && !bad); - CHECK(pure_draft_splice(drafts[0], 2, 0, (PureSpan){(const unsigned char *)"+1", 2}) == PURE_OK); - CHECK(pure_draft_splice(drafts[1], 0, 2, (PureSpan){(const unsigned char *)"B", 1}) == PURE_OK); - CHECK(pure_draft_splice(drafts[2], 0, 0, (PureSpan){(const unsigned char *)"state", 5}) == PURE_OK); - Oracle alpha = {(unsigned char *)"a0", 2, 1}, beta = {(unsigned char *)"b0", 2, 1}; - Oracle alpha_new = {(unsigned char *)"a0+1", 4, 1}, beta_new = {(unsigned char *)"B", 1, 1}; - Oracle state_new = {(unsigned char *)"state", 5, 0}; - /* A process that dies anywhere in the commit leaves all three as before, - * or all three published once the head is written. */ - for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { - pid_t child = fork(); - CHECK(child >= 0); - if (!child) { - crash = 1; - fail_stage = stage; - pure_draft_commit_all(drafts, 3, revisions); - _exit(0); - } - int status; - CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); - PureDraft *state; - CHECK(pure_state_open(path, "values", &state) == PURE_OK); - if (stage < PURE_BEFORE_DIR_SYNC) { - expect(path, "alpha", &alpha); - expect(path, "beta", &beta); - CHECK(pure_draft_revision(state) == 0 && pure_draft_size(state) == 0); - pure_draft_close(state); - continue; - } - Oracle a = alpha_new, b = beta_new, v = state_new; - a.revision = b.revision = 2; - v.revision = 1; - expect(path, "alpha", &a); - expect(path, "beta", &b); - expect_draft(state, &v, scratch); - pure_draft_close(state); - /* Undo the crashed commit's effect for the next checks. */ - CHECK(pure_store_put(path, "alpha", 2, (PureSpan){(const unsigned char *)"a0", 2}, &revision) == PURE_OK); - } - /* The same batch in this process is now stale on every key. */ - CHECK(pure_draft_commit_all(drafts, 3, revisions) == PURE_CONFLICT && revisions[0] == 3); - for (unsigned i = 0; i < 3; ++i) pure_draft_close(drafts[i]); - CHECK(pure_draft_open(path, "alpha", &drafts[0]) == PURE_OK && pure_draft_open(path, "beta", &drafts[1]) == PURE_OK); - CHECK(pure_state_open(path, "values", &drafts[2]) == PURE_OK); - CHECK(pure_draft_splice(drafts[0], 0, 2, (PureSpan){(const unsigned char *)"A", 1}) == PURE_OK); - CHECK(pure_draft_splice(drafts[1], 1, 0, (PureSpan){(const unsigned char *)"!", 1}) == PURE_OK); - CHECK(pure_draft_splice(drafts[2], 5, 0, (PureSpan){(const unsigned char *)"+", 1}) == PURE_OK); - alpha = (Oracle){(unsigned char *)"a0", 2, 3}; - beta = (Oracle){(unsigned char *)"B", 1, 2}; - uint64_t size = file_size(path); - for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; ++fail_stage) { - CHECK(pure_draft_commit_all(drafts, 3, revisions) == PURE_STORE_ERROR); - expect(path, "alpha", &alpha); - expect(path, "beta", &beta); - CHECK(file_size(path) == size); - } - fail_stage = -1; - /* One stale draft stops the whole batch. */ - PureDraft *other; - CHECK(pure_draft_open(path, "beta", &other) == PURE_OK); - CHECK(pure_draft_splice(other, 0, 0, (PureSpan){(const unsigned char *)"x", 1}) == PURE_OK); - CHECK(pure_draft_commit(other, &revision) == PURE_OK && revision == 3); - pure_draft_close(other); - CHECK(pure_draft_commit_all(drafts, 3, revisions) == PURE_CONFLICT && revisions[1] == 3); - expect(path, "alpha", &alpha); - pure_draft_close(drafts[1]); - CHECK(pure_draft_open(path, "beta", &drafts[1]) == PURE_OK); - CHECK(pure_draft_splice(drafts[1], 0, 0, (PureSpan){(const unsigned char *)"y", 1}) == PURE_OK); - CHECK(pure_draft_commit_all(drafts, 3, revisions) == PURE_OK); - CHECK(revisions[0] == 4 && revisions[1] == 4 && revisions[2] == 2); - expect(path, "alpha", &(Oracle){(unsigned char *)"A", 1, 4}); - expect(path, "beta", &(Oracle){(unsigned char *)"yxB", 3, 4}); - expect_draft(drafts[2], &(Oracle){(unsigned char *)"state+", 6, 2}, scratch); - /* Drafts continue from the new revisions. */ - CHECK(pure_draft_splice(drafts[0], 1, 0, (PureSpan){(const unsigned char *)"z", 1}) == PURE_OK); - CHECK(pure_draft_commit_all(drafts, 1, revisions) == PURE_OK && revisions[0] == 5); - /* Invalid batches change nothing. */ - PureDraft *twice[2] = {drafts[0], drafts[0]}, *anonymous = pure_draft_new(); - CHECK(pure_draft_commit_all(twice, 2, revisions) == PURE_INVALID); - PureDraft *mixed[2] = {drafts[1], anonymous}; - CHECK(pure_draft_commit_all(mixed, 2, revisions) == PURE_INVALID); - pure_draft_close(anonymous); - for (unsigned i = 0; i < 3; ++i) pure_draft_close(drafts[i]); - /* Runtime values are not documents: listings skip them and document - * operations refuse their names. */ - char seen[256] = ""; - CHECK(pure_store_keys(path, list_key, seen) == PURE_OK && !strcmp(seen, "alpha,beta,")); - CHECK(pure_store_put(path, ".values", 0, (PureSpan){(const unsigned char *)"x", 1}, &revision) != PURE_OK); - /* Many keys, in no particular order, in one commit. */ - PureDraft *many[50]; - int64_t many_revisions[50]; - char key[16]; - for (unsigned i = 0; i < 50; ++i) { - snprintf(key, sizeof(key), "k%02u", (i * 37) % 50); - CHECK((i % 2 ? pure_state_open(path, key, &many[i]) : pure_draft_open(path, key, &many[i])) == PURE_OK); - CHECK(pure_draft_splice(many[i], 0, 0, (PureSpan){(const unsigned char *)key, strlen(key)}) == PURE_OK); - } - CHECK(pure_draft_commit_all(many, 50, many_revisions) == PURE_OK); - for (unsigned i = 0; i < 50; ++i) { - CHECK(many_revisions[i] == 1); - snprintf(key, sizeof(key), "k%02u", (i * 37) % 50); - PureDraft *check; - CHECK((i % 2 ? pure_state_open(path, key, &check) : pure_draft_open(path, key, &check)) == PURE_OK); - expect_draft(check, &(Oracle){(unsigned char *)key, strlen(key), 1}, scratch); - pure_draft_close(check); - pure_draft_close(many[i]); - } -} - -/* A store handle shares one mapping among its drafts, sees commits made by - * other writers when a draft opens, follows a compaction that replaced the - * file, and maps a batch it committed once. */ -static void store_handle(const char *path) { - unsigned char scratch[64]; - int64_t revision, revisions[3]; - CHECK(pure_store_put(path, "shared", 0, (PureSpan){(const unsigned char *)"one", 3}, &revision) == PURE_OK); - PureStore *store; - CHECK(pure_store_open(path, &store) == PURE_OK); - PureDraft *a, *b, *state, *bad; - CHECK(pure_store_draft(store, "shared", &a) == PURE_OK); - expect_draft(a, &(Oracle){(unsigned char *)"one", 3, 1}, scratch); - CHECK(pure_store_draft(store, "bad.key", &bad) == PURE_INVALID && !bad); - /* Another writer commits: the next draft sees it, the open one keeps its version. */ - CHECK(pure_store_put(path, "shared", 1, (PureSpan){(const unsigned char *)"two", 3}, &revision) == PURE_OK); - CHECK(pure_store_draft(store, "shared", &b) == PURE_OK); - expect_draft(b, &(Oracle){(unsigned char *)"two", 3, 2}, scratch); - expect_draft(a, &(Oracle){(unsigned char *)"one", 3, 1}, scratch); - CHECK(pure_draft_splice(a, 3, 0, (PureSpan){(const unsigned char *)"!", 1}) == PURE_OK); - CHECK(pure_draft_commit(a, &revision) == PURE_CONFLICT && revision == 2); - pure_draft_close(a); - /* A batch of handle drafts, a runtime value among them. */ - CHECK(pure_store_state(store, "values", &state) == PURE_OK && pure_draft_revision(state) == 0); - CHECK(pure_store_draft(store, "other", &a) == PURE_OK); - CHECK(pure_draft_splice(b, 3, 0, (PureSpan){(const unsigned char *)"+", 1}) == PURE_OK); - CHECK(pure_draft_splice(state, 0, 0, (PureSpan){(const unsigned char *)"v", 1}) == PURE_OK); - CHECK(pure_draft_splice(a, 0, 0, (PureSpan){(const unsigned char *)"o", 1}) == PURE_OK); - PureDraft *batch[3] = {b, state, a}; - CHECK(pure_draft_commit_all(batch, 3, revisions) == PURE_OK && revisions[0] == 3 && revisions[1] == 1); - expect(path, "shared", &(Oracle){(unsigned char *)"two+", 4, 3}); - expect_draft(state, &(Oracle){(unsigned char *)"v", 1, 1}, scratch); - /* Other writers grow the file past compaction: it is replaced under the handle. */ - unsigned char *big = malloc(100000); - uint32_t seed = 5; - CHECK(big); - fill(big, 100000, &seed); - int64_t filler = 0; - char name[512]; - struct stat before, replaced; - snprintf(name, sizeof(name), "%s/pure.tree", path); - CHECK(stat(name, &before) == 0); - for (int i = 0; i < 8; ++i) CHECK(pure_store_put(path, "filler", filler, (PureSpan){big, 100000}, &filler) == PURE_OK); - CHECK(stat(name, &replaced) == 0 && replaced.st_ino != before.st_ino); - free(big); - PureDraft *after; - CHECK(pure_store_draft(store, "shared", &after) == PURE_OK); - expect_draft(after, &(Oracle){(unsigned char *)"two+", 4, 3}, scratch); - /* Drafts of the old file still read and commit (their extents are copied). */ - expect_draft(b, &(Oracle){(unsigned char *)"two+", 4, 3}, scratch); - CHECK(pure_draft_splice(b, 0, 0, (PureSpan){(const unsigned char *)">", 1}) == PURE_OK); - CHECK(pure_draft_commit(b, &revision) == PURE_OK && revision == 4); - expect(path, "shared", &(Oracle){(unsigned char *)">two+", 5, 4}); - pure_draft_close(after); - pure_draft_close(a); - pure_draft_close(b); - pure_draft_close(state); - pure_store_close(store); -} - -/* Threads of one process commit through their own handles at once: none is - * refused as busy, every commit lands, and edits of one shared key conflict - * instead of overwriting each other. */ -typedef struct { const char *path; unsigned number, conflicts; } Committer; - -static void *commit_many(void *context) { - Committer *c = context; - PureStore *store; - CHECK(pure_store_open(c->path, &store) == PURE_OK); - char key[16]; - snprintf(key, sizeof(key), "t%u", c->number); - for (unsigned i = 0; i < 40; ++i) { - PureDraft *own, *shared; - int64_t revisions[2]; - CHECK(pure_store_draft(store, key, &own) == PURE_OK && pure_store_draft(store, "shared", &shared) == PURE_OK); - CHECK(pure_draft_splice(own, pure_draft_size(own), 0, (PureSpan){(const unsigned char *)"x", 1}) == PURE_OK); - CHECK(pure_draft_splice(shared, pure_draft_size(shared), 0, (PureSpan){(const unsigned char *)"y", 1}) == PURE_OK); - PureDraft *batch[2] = {own, shared}; - PureResult result = pure_draft_commit_all(batch, 2, revisions); - CHECK(result == PURE_OK || result == PURE_CONFLICT); - if (result == PURE_CONFLICT) { - ++c->conflicts; - CHECK(pure_draft_commit(own, &revisions[0]) == PURE_OK); - } - pure_draft_close(own); - pure_draft_close(shared); - } - pure_store_close(store); - return NULL; -} - -static void threads(const char *path) { - pthread_t thread[4]; - Committer committer[4]; - for (unsigned i = 0; i < 4; ++i) { - committer[i] = (Committer){path, i, 0}; - CHECK(pthread_create(&thread[i], NULL, commit_many, &committer[i]) == 0); - } - unsigned conflicts = 0; - for (unsigned i = 0; i < 4; ++i) { - CHECK(pthread_join(thread[i], NULL) == 0); - conflicts += committer[i].conflicts; - } - PureStoredView view; - char key[16]; - for (unsigned i = 0; i < 4; ++i) { - snprintf(key, sizeof(key), "t%u", i); - CHECK(pure_store_get(path, key, PURE_OBJECT_LIMIT, &view) == PURE_OK && view.body.size == 40 && view.revision == 40); - pure_store_release(&view); - } - CHECK(pure_store_get(path, "shared", PURE_OBJECT_LIMIT, &view) == PURE_OK); - CHECK(view.body.size == 160 - conflicts && view.revision == (int64_t)(160 - conflicts)); - pure_store_release(&view); -} - -/* Commits that arrive while another is being published share the next - * publication: one head for all of them. */ -typedef struct { const char *path; unsigned number; PureResult result; } Single; - -static void *commit_one(void *context) { - Single *s = context; - PureStore *store; - PureDraft *draft; - char key[16]; - int64_t revision; - snprintf(key, sizeof(key), "g%u", s->number); - CHECK(pure_store_open(s->path, &store) == PURE_OK && pure_store_draft(store, key, &draft) == PURE_OK); - CHECK(pure_draft_splice(draft, 0, 0, (PureSpan){(const unsigned char *)key, strlen(key)}) == PURE_OK); - s->result = pure_draft_commit(draft, &revision); - CHECK(s->result != PURE_OK || revision == 1); - pure_draft_close(draft); - pure_store_close(store); - return NULL; -} - -static void grouped(const char *path) { - pthread_t thread[5]; - Single single[5]; - unsigned long before = pure_tree_test_publications(); - __atomic_store_n(&slow_once, 1, __ATOMIC_SEQ_CST); - for (unsigned i = 0; i < 5; ++i) { - single[i] = (Single){path, i, PURE_INVALID}; - CHECK(pthread_create(&thread[i], NULL, commit_one, &single[i]) == 0); - if (!i) pause_ms(20); /* the first is now publishing, slowly */ - } - for (unsigned i = 0; i < 5; ++i) CHECK(pthread_join(thread[i], NULL) == 0 && single[i].result == PURE_OK); - CHECK(pure_tree_test_publications() - before == 2); - char key[16]; - for (unsigned i = 0; i < 5; ++i) { - snprintf(key, sizeof(key), "g%u", i); - expect(path, key, &(Oracle){(unsigned char *)key, strlen(key), 1}); - } -} - -/* A crash during a commit's only sync can leave its head on the device - * without all of its bytes. A process that starts after it passes over that - * head to the previous commit, and the next writer overwrites both. */ -static void lost_commit(const char *path) { - unsigned char first[] = "first", second[] = "second"; - int64_t revision; - CHECK(pure_store_put(path, "lost", 0, (PureSpan){first, 5}, &revision) == PURE_OK); - Oracle o = {first, 5, revision}; - uint64_t start = published_end(path, "lost"); - CHECK(pure_store_put(path, "lost", o.revision, (PureSpan){second, 6}, &revision) == PURE_OK); - uint64_t end = published_end(path, "lost"); - char name[512]; - snprintf(name, sizeof(name), "%s/pure.tree", path); - int fd = open(name, O_RDWR); - off_t at = (off_t)(start + (end - start) / 2); - unsigned char byte; - CHECK(fd >= 0 && pread(fd, &byte, 1, at) == 1); - byte ^= 0xff; - CHECK(pwrite(fd, &byte, 1, at) == 1 && close(fd) == 0); - pure_tree_test_forget(); - expect(path, "lost", &o); - PureStore *store; - PureDraft *draft; - CHECK(pure_store_open(path, &store) == PURE_OK && pure_store_draft(store, "lost", &draft) == PURE_OK); - CHECK(pure_draft_revision(draft) == o.revision); - pure_draft_close(draft); - CHECK(pure_store_put(path, "lost", o.revision + 1, (PureSpan){first, 1}, &revision) == PURE_CONFLICT); - CHECK(pure_store_put(path, "lost", o.revision, (PureSpan){second, 3}, &revision) == PURE_OK); - o = (Oracle){second, 3, revision}; - CHECK(published_end(path, "lost") < end); /* it started where the lost commit did */ - CHECK(pure_store_draft(store, "lost", &draft) == PURE_OK && pure_draft_revision(draft) == o.revision); - pure_draft_close(draft); - pure_store_close(store); - pure_tree_test_forget(); - expect(path, "lost", &o); -} - -/* While a commit's sync is pending, this process keeps reading the previous - * version: a crash could still take the new one back. */ -static void *commit_held(void *context) { - int64_t revision; - CHECK(pure_store_put(context, "pending", 1, (PureSpan){(const unsigned char *)"new", 3}, &revision) == PURE_OK); - return NULL; -} - -static void pending_head(const char *path) { - int64_t revision; - CHECK(pure_store_put(path, "pending", 0, (PureSpan){(const unsigned char *)"old", 3}, &revision) == PURE_OK); - PureStore *store; - PureDraft *draft; - CHECK(pure_store_open(path, &store) == PURE_OK); - __atomic_store_n(&held_sync, 1, __ATOMIC_SEQ_CST); - pthread_t writer; - CHECK(pthread_create(&writer, NULL, commit_held, (void *)path) == 0); - while (__atomic_load_n(&held_sync, __ATOMIC_SEQ_CST) != 2) pause_ms(1); - CHECK(pure_store_draft(store, "pending", &draft) == PURE_OK && pure_draft_revision(draft) == 1); - pure_draft_close(draft); - expect(path, "pending", &(Oracle){(unsigned char *)"old", 3, 1}); - __atomic_store_n(&held_sync, 0, __ATOMIC_SEQ_CST); - CHECK(pthread_join(writer, NULL) == 0); - CHECK(pure_store_draft(store, "pending", &draft) == PURE_OK && pure_draft_revision(draft) == 2); - pure_draft_close(draft); - pure_store_close(store); - expect(path, "pending", &(Oracle){(unsigned char *)"new", 3, 2}); -} - -/* A commit costs one sync, and commits into the written zeros after the end - * leave the file size alone (no journal commit for a new size). */ -static void one_sync(const char *path) { - int64_t revision; - CHECK(pure_store_put(path, "sync", 0, (PureSpan){(const unsigned char *)"0", 1}, &revision) == PURE_OK); - uint64_t size = file_size(path); - unsigned long syncs = pure_tree_test_syncs(); - for (unsigned i = 0; i < 20; ++i) - CHECK(pure_store_put(path, "sync", revision, (PureSpan){(const unsigned char *)"x", 1}, &revision) == PURE_OK); - CHECK(pure_tree_test_syncs() - syncs == 20 && file_size(path) == size); -} - -static void clean(const char *path) { - DIR *entries = opendir(path); - CHECK(entries); - struct dirent *entry; - while ((entry = readdir(entries))) { - if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue; - CHECK(strncmp(entry->d_name, ".pending-", 9)); - CHECK(unlinkat(dirfd(entries), entry->d_name, 0) == 0); - } - CHECK(closedir(entries) == 0 && rmdir(path) == 0); -} - -int main(void) { - void (*checks[])(const char *) = {oracle, amplification, faults, crashes, torn_head, - isolation, corruption, many_keys, revision_limit, keys, - drafts, typing, draft_conflict, draft_relocation, draft_faults, - draft_versions, anonymous_draft, batch, store_handle, threads, grouped, - lost_commit, pending_head, one_sync, records}; - for (size_t i = 0; i < sizeof(checks) / sizeof(*checks); ++i) { - char path[] = "/tmp/pure-tree-check-XXXXXX"; - CHECK(mkdtemp(path) && pure_store_init(path) == PURE_OK); - checks[i](path); - CHECK(pure_store_init(path) == PURE_OK); - clean(path); - } - puts("{\"tree_store\":\"random-oracle-balance-held-views-one-byte-amplification-faults-" - "process-crash-torn-head-lockless-isolation-compaction-corruption-1024-keys-revision-limit-key-listing-" - "draft-oracle-typing-arena-conflict-relocation-commit-faults-held-versions-anonymous-" - "atomic-batch-runtime-keys-store-handle-threads-group-commit-lost-commit-pending-head-one-sync-records\"}"); - return 0; -} diff --git a/research/experiments/009-runtime-drafts/run.py b/research/experiments/009-runtime-drafts/run.py index 08ed55d..eb87a25 100755 --- a/research/experiments/009-runtime-drafts/run.py +++ b/research/experiments/009-runtime-drafts/run.py @@ -8,12 +8,12 @@ import subprocess from pathlib import Path HERE = Path(__file__).resolve().parent -CONNECTIONS = HERE.parents[2] / 'prototypes' / 'connections' +ROOT = HERE.parents[2] def main(): parser = argparse.ArgumentParser() - parser.add_argument('--work', type=Path, default=CONNECTIONS / 'build' / 'bench-drafts') + parser.add_argument('--work', type=Path, default=ROOT / 'build' / 'bench-drafts') parser.add_argument('--size', type=int, default=1048576) parser.add_argument('--saves', type=int, default=24) parser.add_argument('--rounds', type=int, default=3) @@ -24,8 +24,8 @@ def main(): device = subprocess.check_output(['findmnt', '-n', '-T', str(args.work), '-o', 'SOURCE,FSTYPE'], text=True).split() assert device[1] != 'tmpfs', 'fsync is free on tmpfs' binary = args.work / 'bench' - subprocess.run(['cc', '-std=c17', '-O2', '-Wall', '-Wextra', '-Werror', f'-I{CONNECTIONS}', - str(HERE / 'bench.c'), str(CONNECTIONS / 'store-tree.c'), '-o', str(binary)], check=True) + subprocess.run(['cc', '-std=c17', '-O2', '-Wall', '-Wextra', '-Werror', f'-I{ROOT / "core"}', + str(HERE / 'bench.c'), str(ROOT / 'store' / 'store-tree.c'), '-o', str(binary)], check=True) runs = [] for round_number in range(args.rounds): for mode in (('copy', 'draft') if round_number % 2 == 0 else ('draft', 'copy')): diff --git a/research/experiments/010-one-sync-commit/run.py b/research/experiments/010-one-sync-commit/run.py index 9a01c07..c729358 100644 --- a/research/experiments/010-one-sync-commit/run.py +++ b/research/experiments/010-one-sync-commit/run.py @@ -16,11 +16,18 @@ from pathlib import Path HERE = Path(__file__).resolve().parent ROOT = HERE.parents[2] -PURE = ROOT / 'prototypes' / 'connections' +CORE = ROOT / 'core' # the store's headers +STORE = ROOT / 'store' / 'store-tree.c' + + +def stored_at(revision): + """Where the tree store was at a revision: store/ now, prototypes/connections/ before.""" + moved = subprocess.run(['git', '-C', str(ROOT), 'cat-file', '-e', f'{revision}:store/store-tree.c'], capture_output=True) + return 'store/store-tree.c' if moved.returncode == 0 else 'prototypes/connections/store-tree.c' def build(output, sources): - subprocess.run(['cc', '-O2', '-std=c17', f'-I{PURE}', '-o', str(output)] + [str(s) for s in sources] + ['-lpthread'], + subprocess.run(['cc', '-O2', '-std=c17', f'-I{CORE}', '-o', str(output)] + [str(s) for s in sources] + ['-lpthread'], check=True) @@ -34,12 +41,12 @@ def main(): with tempfile.TemporaryDirectory() as work: work = Path(work) before = work / 'store-tree-before.c' - before.write_text(subprocess.run(['git', '-C', str(ROOT), 'show', f'{args.before}:prototypes/connections/store-tree.c'], + before.write_text(subprocess.run(['git', '-C', str(ROOT), 'show', f'{args.before}:{stored_at(args.before)}'], capture_output=True, text=True, check=True).stdout) build(work / 'probe', [HERE / 'probe.c']) builds = {'two syncs': work / 'commits-before', 'one sync': work / 'commits-after'} build(builds['two syncs'], [HERE / 'commits.c', before]) - build(builds['one sync'], [HERE / 'commits.c', PURE / 'store-tree.c']) + build(builds['one sync'], [HERE / 'commits.c', STORE]) results = {'rounds': []} for round_ in range(args.rounds): probe = subprocess.run([str(work / 'probe'), str(args.directory)], capture_output=True, text=True, check=True) diff --git a/research/experiments/012-tail-records/run.py b/research/experiments/012-tail-records/run.py index 7fc342e..ed8630b 100755 --- a/research/experiments/012-tail-records/run.py +++ b/research/experiments/012-tail-records/run.py @@ -18,12 +18,19 @@ from pathlib import Path HERE = Path(__file__).resolve().parent ROOT = HERE.parents[2] -PURE = ROOT / 'prototypes' / 'connections' +CORE = ROOT / 'core' # the store's headers +STORE = ROOT / 'store' / 'store-tree.c' COMMITS = HERE.parent / '010-one-sync-commit' / 'commits.c' +def stored_at(revision): + """Where the tree store was at a revision: store/ now, prototypes/connections/ before.""" + moved = subprocess.run(['git', '-C', str(ROOT), 'cat-file', '-e', f'{revision}:store/store-tree.c'], capture_output=True) + return 'store/store-tree.c' if moved.returncode == 0 else 'prototypes/connections/store-tree.c' + + def build(output, sources): - subprocess.run(['cc', '-O2', '-std=c17', f'-I{PURE}', '-o', str(output)] + [str(s) for s in sources] + ['-lpthread'], + subprocess.run(['cc', '-O2', '-std=c17', f'-I{CORE}', '-o', str(output)] + [str(s) for s in sources] + ['-lpthread'], check=True) @@ -37,12 +44,12 @@ def main(): with tempfile.TemporaryDirectory() as work: work = Path(work) before = work / 'store-tree-before.c' - before.write_text(subprocess.run(['git', '-C', str(ROOT), 'show', f'{args.before}:prototypes/connections/store-tree.c'], + before.write_text(subprocess.run(['git', '-C', str(ROOT), 'show', f'{args.before}:{stored_at(args.before)}'], capture_output=True, text=True, check=True).stdout) build(work / 'probe', [HERE / 'probe.c']) builds = {'heads in slots': work / 'commits-before', 'heads as records': work / 'commits-after'} build(builds['heads in slots'], [COMMITS, before]) - build(builds['heads as records'], [COMMITS, PURE / 'store-tree.c']) + build(builds['heads as records'], [COMMITS, STORE]) results = {'rounds': []} for round_ in range(args.rounds): measured = {'probe': [], 'commits': {}} diff --git a/serve/deliver.c b/serve/deliver.c new file mode 100644 index 0000000..0d3da64 --- /dev/null +++ b/serve/deliver.c @@ -0,0 +1,716 @@ +#define _POSIX_C_SOURCE 200809L +#include "deliver.h" +#include "message-json.h" +#include "module-text.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { TIMEOUT_S = 10, FIRST_PAUSE_MS = 100, LONGEST_PAUSE_MS = 10000, GROUPS = PURE_MODULE_NAME / 2, BATCH_MOST = 256, + ANSWER_MOST = 1 << 20 }; +enum { DELIVERED, AGAIN, REFUSED_BY_PEER, TLS_FAILED, ANSWER_UNREAD }; + +typedef struct { char *body; size_t size; uint64_t seq; unsigned char *sent; size_t sent_size; } Pending; +/* An answer made into its input's message: a table of the fields' starts and + * sizes (u32le), then the payload from its prefix on. */ +typedef struct { uint64_t seq; unsigned char *bytes; size_t size, table; } Answer; + +struct PureWire { + const PureModule *module; + uint32_t output; + char host[256], port[8], path[1024], source[PURE_SOURCE + 1]; + pthread_mutex_t lock; + pthread_cond_t changed; + Pending *queue; /* the sends to deliver, in order, from head */ + size_t head, count, capacity; + uint64_t queued, delivered, attempts, deliveries; + unsigned batch; /* how many a request may carry, as the receiver said (X-Pure-Batch); 0: one */ + bool stopping, started, stuck; + pthread_t thread; + void (*notify)(void *); + void *context; + int fd; /* the connection kept for the next send, or -1; changed under the lock */ + bool tls; /* https: the connection is TLS */ + SSL *ssl; /* its TLS session */ + int answer_input; /* the input answers become events of, or -1 */ + Answer *answers; /* made, for the owner to take in order */ + size_t answer_head, answer_count, answer_capacity; + Answer taken; /* the one the owner took last, kept until it takes the next */ + uint32_t placed[2 * PURE_MODULE_FIELDS]; +}; + +/* ---- TLS: one client context, verifying the receiver's certificate against + * the system's CAs or the file pure_wire_trust named, and its name. */ +static pthread_once_t tls_once = PTHREAD_ONCE_INIT; +static SSL_CTX *tls_context; +static const char *tls_ca; + +void pure_wire_trust(const char *ca_file) { tls_ca = ca_file; } + +static void make_tls_context(void) { + SSL_CTX *context = SSL_CTX_new(TLS_client_method()); + if (!context) return; + if (!SSL_CTX_set_min_proto_version(context, TLS1_2_VERSION) || + !(tls_ca ? SSL_CTX_load_verify_locations(context, tls_ca, NULL) : SSL_CTX_set_default_verify_paths(context))) { + SSL_CTX_free(context); + return; + } + SSL_CTX_set_verify(context, SSL_VERIFY_PEER, NULL); + SSL_CTX_set_mode(context, SSL_MODE_AUTO_RETRY); + tls_context = context; +} + +/* ---- A send as JSON: its fields in order, a group's fields as an object. */ + +typedef struct { char *bytes; size_t size, capacity; bool failed; } Text; + +static void put(Text *t, const void *bytes, size_t size) { + if (t->failed) return; + if (size > t->capacity - t->size) { + size_t capacity = t->capacity ? t->capacity : 256; + while (size > capacity - t->size) capacity *= 2; + char *grown = realloc(t->bytes, capacity); + if (!grown) { t->failed = true; return; } + t->bytes = grown; + t->capacity = capacity; + } + memcpy(t->bytes + t->size, bytes, size); + t->size += size; +} + +static void put_string(Text *t, const unsigned char *bytes, size_t size) { + put(t, "\"", 1); + size_t run = 0; + for (size_t i = 0; i < size; ++i) { + unsigned char b = bytes[i]; + if (b != '"' && b != '\\' && b >= 0x20) continue; + put(t, bytes + run, i - run); + char escaped[8]; + if (b == '"' || b == '\\') put(t, (char[]){'\\', (char)b}, 2); + else put(t, escaped, (size_t)snprintf(escaped, sizeof(escaped), "\\u%04x", b)); + run = i + 1; + } + put(t, bytes + run, size - run); + put(t, "\"", 1); +} + +static uint32_t read_u32(const unsigned char *p) { return p[0] | p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; } + +/* The group a field is in: the length of its name before the last dot (0: none). */ +static size_t group_of(const char *name) { + const char *dot = strrchr(name, '.'); + return dot ? (size_t)(dot - name) : 0; +} + +/* Whether the group path open (length open_size) holds the group of name (length size). */ +static bool holds(const char *open, size_t open_size, const char *name, size_t size) { + return open_size <= size && !memcmp(open, name, open_size) && (open_size == size || !open_size || name[open_size] == '.'); +} + +static bool encode(const PureWire *w, PureSpan payload, Text *t) { + const PureModule *m = w->module; + const PureInputShape *shape = &m->input[w->output]; + if (payload.size < shape->prefix) return false; + char open[PURE_MODULE_NAME] = ""; + size_t open_size = 0, at = shape->prefix; + unsigned depth = 0; + bool first[GROUPS + 1] = {true}; + put(t, "{", 1); + for (uint32_t f = 0; f < shape->field_count; ++f) { + const PureField *field = &m->field[shape->field_start + f]; + size_t group = group_of(field->name); + while (depth && !holds(open, open_size, field->name, group)) { + put(t, "}", 1); + --depth; + open_size = group_of(open) < open_size ? group_of(open) : 0; + open[open_size] = 0; + } + while (open_size < group) { + size_t start = open_size ? open_size + 1 : 0, end = start; + while (end < group && field->name[end] != '.') ++end; + if (!first[depth]) put(t, ",", 1); + first[depth] = false; + put_string(t, (const unsigned char *)field->name + start, end - start); + put(t, ":{", 2); + first[++depth] = true; + memcpy(open, field->name, end); + open[open_size = end] = 0; + } + if (!first[depth]) put(t, ",", 1); + first[depth] = false; + const char *key = field->name + (group ? group + 1 : 0); + put_string(t, (const unsigned char *)key, strlen(key)); + put(t, ":", 1); + char digits[32]; + if (field->type == PURE_FIELD_U32) { + put(t, digits, (size_t)snprintf(digits, sizeof(digits), "%u", (unsigned)read_u32(payload.data + field->offset))); + } else if (field->type == PURE_FIELD_NUMBER) { + uint64_t bits = read_u32(payload.data + field->offset) | (uint64_t)read_u32(payload.data + field->offset + 4) << 32; + double value; + memcpy(&value, &bits, sizeof(value)); + put(t, digits, pure_format_number(value, digits)); + } else { + size_t size = field->offset == PURE_FIELD_REST ? payload.size - at : read_u32(payload.data + field->offset); + if (size > payload.size - at) return false; + PureSpan value = {payload.data + at, size}; + if (!pure_utf8_valid(value)) return false; /* bytes that are not text have no JSON form */ + put_string(t, value.data, size); + at += size; + } + } + while (depth--) put(t, "}", 1); + put(t, "}", 1); + return !t->failed; +} + +/* ---- The wire. */ + +PureWire *pure_wire_open(const PureModule *m, const char *output, const char *url, const char *source, + char *error, size_t size) { + int index = pure_module_input(m, output); + if (index < 0 || m->input[index].kind != PURE_KIND_OUTPUT) { + snprintf(error, size, "no output '%s'", output); + return NULL; + } + /* A group's fields are together, so each group is one object. */ + const PureInputShape *shape = &m->input[index]; + for (uint32_t f = 1; f < shape->field_count; ++f) { + const char *name = m->field[shape->field_start + f].name; + for (uint32_t e = 0; e + 1 < f; ++e) { + const char *earlier = m->field[shape->field_start + e].name, *between = m->field[shape->field_start + f - 1].name; + size_t shared = 0; + while (earlier[shared] && earlier[shared] == name[shared]) ++shared; + while (shared && earlier[shared - 1] != '.') --shared; /* the groups both are in */ + if (shared && strncmp(between, name, shared)) { + snprintf(error, size, "the fields of output '%s' in one group are not together", output); + return NULL; + } + } + } + size_t length = strlen(source); + if (!length || length > PURE_SOURCE) { + snprintf(error, size, "a source name has 1 to %d bytes", PURE_SOURCE); + return NULL; + } + bool tls = !strncmp(url, "https://", 8); + if (!tls && strncmp(url, "http://", 7)) { + snprintf(error, size, "a URL is http[s]://host[:port]/path"); + return NULL; + } + if (tls) { + pthread_once(&tls_once, make_tls_context); + if (!tls_context) { + snprintf(error, size, "no TLS context: %s cannot be read", tls_ca ? tls_ca : "the system's CAs"); + return NULL; + } + } + const char *host = url + (tls ? 8 : 7), *slash = strchr(host, '/'), *colon = memchr(host, ':', slash ? (size_t)(slash - host) : strlen(host)); + const char *host_end = colon ? colon : slash ? slash : host + strlen(host); + const char *port = colon ? colon + 1 : tls ? "443" : "80"; + size_t port_size = colon ? (size_t)((slash ? slash : host + strlen(host)) - port) : strlen(port); + if (host_end == host || (size_t)(host_end - host) >= 256 || !port_size || port_size >= 8 || + (slash && strlen(slash) >= 1024)) { + snprintf(error, size, "a URL is http://host[:port]/path"); + return NULL; + } + PureWire *w = calloc(1, sizeof(*w)); + if (!w) { snprintf(error, size, "out of memory"); return NULL; } + w->module = m; + w->output = (uint32_t)index; + memcpy(w->host, host, (size_t)(host_end - host)); + memcpy(w->port, port, port_size); + snprintf(w->path, sizeof(w->path), "%s", slash ? slash : "/"); + memcpy(w->source, source, length + 1); + w->fd = -1; + w->tls = tls; + w->answer_input = -1; + pthread_mutex_init(&w->lock, NULL); + pthread_cond_init(&w->changed, NULL); + return w; +} + +uint32_t pure_wire_output(const PureWire *w) { return w->output; } + +bool pure_wire_reply(PureWire *w, uint32_t input, char *error, size_t size) { + const PureModule *m = w->module; + if (input >= m->inputs || m->input[input].kind != PURE_KIND_MESSAGE || m->input[input].keyed) { + snprintf(error, size, "an answer goes to a message input without a key"); + return false; + } + w->answer_input = (int)input; + return true; +} + +bool pure_wire_answer(PureWire *w, PureWireAnswer *out) { + free(w->taken.bytes); + w->taken = (Answer){0}; + pthread_mutex_lock(&w->lock); + bool some = w->answer_count > 0; + if (some) { + w->taken = w->answers[w->answer_head++]; + --w->answer_count; + w->delivered = w->taken.seq; /* trimmed after the owner applies it */ + } + pthread_mutex_unlock(&w->lock); + if (!some) return false; + const PureInputShape *in = &w->module->input[w->answer_input]; + for (size_t i = 0; i < 2 * (size_t)in->field_count; ++i) w->placed[i] = read_u32(w->taken.bytes + 4 * i); + *out = (PureWireAnswer){(uint32_t)w->answer_input, w->taken.seq, + {w->taken.bytes + w->taken.table, w->taken.size - w->taken.table}, w->placed, w->source}; + return true; +} +uint64_t pure_wire_queued(const PureWire *w) { return w->queued; } + +uint64_t pure_wire_delivered(PureWire *w) { + pthread_mutex_lock(&w->lock); + uint64_t delivered = w->delivered; + pthread_mutex_unlock(&w->lock); + return delivered; +} + +void pure_wire_counts(PureWire *w, uint64_t *delivered, uint64_t *attempts) { + pthread_mutex_lock(&w->lock); + *delivered = w->deliveries; + *attempts = w->attempts; + pthread_mutex_unlock(&w->lock); +} + +bool pure_wire_queue(PureWire *w, uint64_t seq, PureSpan payload) { + if (w->stuck) return false; + Text t = {0}; + if (!encode(w, payload, &t)) { + free(t.bytes); + w->stuck = true; + fprintf(stderr, "{\"wire\":\"%s\",\"seq\":%llu,\"error\":\"no JSON form: its bytes are not text\"}\n", w->source, + (unsigned long long)seq); + return false; + } + pthread_mutex_lock(&w->lock); + if (w->head + w->count == w->capacity) { + if (w->head) { + memmove(w->queue, w->queue + w->head, w->count * sizeof(*w->queue)); + w->head = 0; + } else { + size_t capacity = w->capacity ? 2 * w->capacity : 16; + Pending *grown = realloc(w->queue, capacity * sizeof(*grown)); + if (!grown) { + pthread_mutex_unlock(&w->lock); + free(t.bytes); + return false; /* it stays in the outbox: queued after the next save */ + } + w->queue = grown; + w->capacity = capacity; + } + } + unsigned char *sent = NULL; + if (w->answer_input >= 0 && !(sent = malloc(payload.size ? payload.size : 1))) { + pthread_mutex_unlock(&w->lock); + free(t.bytes); + return false; + } + if (sent && payload.size) memcpy(sent, payload.data, payload.size); + w->queue[w->head + w->count++] = (Pending){t.bytes, t.size, seq, sent, payload.size}; + w->queued = seq; + pthread_cond_signal(&w->changed); + pthread_mutex_unlock(&w->lock); + return true; +} + +/* ---- HTTP/1.1, one request at a time on a kept connection. */ + +typedef struct { PureWire *wire; char bytes[4096]; size_t at, end; } Reader; + +static bool fill(Reader *r) { + if (r->at == r->end) r->at = r->end = 0; + if (r->end == sizeof(r->bytes)) return false; + PureWire *w = r->wire; + ssize_t n; + if (w->ssl) n = SSL_read(w->ssl, r->bytes + r->end, (int)(sizeof(r->bytes) - r->end)); + else do n = recv(w->fd, r->bytes + r->end, sizeof(r->bytes) - r->end, 0); while (n < 0 && errno == EINTR); + if (n <= 0) return false; +#ifdef TCP_QUICKACK + /* Acknowledge at once: a server that writes an answer in pieces holds the + * next piece until the first is acknowledged (Nagle), which a delayed + * acknowledgement would stall for 40 ms. The flag does not stay set. */ + int one = 1; + setsockopt(w->fd, IPPROTO_TCP, TCP_QUICKACK, &one, sizeof(one)); +#endif + r->end += (size_t)n; + return true; +} + +/* A line without its CRLF, in line (at most size - 1 bytes). */ +static bool read_line(Reader *r, char *line, size_t size) { + for (;;) { + char *end = memchr(r->bytes + r->at, '\n', r->end - r->at); + if (end) { + size_t length = (size_t)(end - (r->bytes + r->at)); + if (length && end[-1] == '\r') --length; + if (length >= size) return false; + memcpy(line, r->bytes + r->at, length); + line[length] = 0; + r->at = (size_t)(end - r->bytes) + 1; + return true; + } + if (r->at) { + memmove(r->bytes, r->bytes + r->at, r->end - r->at); + r->end -= r->at; + r->at = 0; + } + if (!fill(r)) return false; + } +} + +/* size bytes of the body: kept in into (up to ANSWER_MOST), or skipped when it is NULL. */ +static bool take(Reader *r, uint64_t size, Text *into) { + if (into && size > ANSWER_MOST - into->size) return false; + while (size) { + if (r->at == r->end && !fill(r)) return false; + size_t n = r->end - r->at < size ? r->end - r->at : (size_t)size; + if (into) put(into, r->bytes + r->at, n); + r->at += n; + size -= n; + } + return !into || !into->failed; +} + +static bool send_all(PureWire *w, const char *bytes, size_t size) { + while (size) { + ssize_t n = w->ssl ? SSL_write(w->ssl, bytes, (int)(size < INT32_MAX ? size : INT32_MAX)) : + send(w->fd, bytes, size, MSG_NOSIGNAL); + if (n < 0 && !w->ssl && errno == EINTR) continue; + if (n <= 0) return false; + bytes += n; + size -= (size_t)n; + } + return true; +} + +/* TLS over a connected socket: the receiver's certificate must name the host + * (a DNS name, or an IP address written as one). */ +static SSL *start_tls(const PureWire *w, int fd) { + SSL *ssl = SSL_new(tls_context); + if (!ssl) return NULL; + unsigned char address[16]; + bool literal = inet_pton(AF_INET, w->host, address) == 1 || inet_pton(AF_INET6, w->host, address) == 1; + bool named = literal ? X509_VERIFY_PARAM_set1_ip_asc(SSL_get0_param(ssl), w->host) == 1 : + SSL_set_tlsext_host_name(ssl, w->host) == 1 && SSL_set1_host(ssl, w->host) == 1; + if (!named || SSL_set_fd(ssl, fd) != 1 || SSL_connect(ssl) != 1 || SSL_get_verify_result(ssl) != X509_V_OK) { + SSL_free(ssl); + return NULL; + } + return ssl; +} + +static int connect_to(const PureWire *w) { + struct addrinfo hints = {.ai_family = AF_UNSPEC, .ai_socktype = SOCK_STREAM}, *list; + if (getaddrinfo(w->host, w->port, &hints, &list)) return -1; + int fd = -1; + for (struct addrinfo *a = list; a && fd < 0; a = a->ai_next) { + fd = socket(a->ai_family, a->ai_socktype | SOCK_CLOEXEC, a->ai_protocol); + if (fd < 0) continue; + struct timeval limit = {TIMEOUT_S, 0}; + int one = 1; + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &limit, sizeof(limit)); + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &limit, sizeof(limit)); + setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + if (connect(fd, a->ai_addr, a->ai_addrlen)) { + close(fd); + fd = -1; + } + } + freeaddrinfo(list); + return fd; +} + +static void drop_connection(PureWire *w) { + pthread_mutex_lock(&w->lock); + if (w->ssl) SSL_free(w->ssl); + if (w->fd >= 0) close(w->fd); + w->ssl = NULL; + w->fd = -1; + pthread_mutex_unlock(&w->lock); +} + +/* A fault a Pure receiver answers when its app refused the message. */ +static bool refusal(const char *error) { + static const char *const faults[] = {"divide", "non_finite", "clamp", "limit", "not_found", "encoding", "exists"}; + for (size_t i = 0; i < sizeof(faults) / sizeof(*faults); ++i) if (!strcmp(error, faults[i])) return true; + return false; +} + +/* The answer to one request: its status, whether a Pure receiver refused it, + * and whether the connection may carry the next. */ +static bool read_answer(PureWire *w, int *status, bool *refused, bool *keep, unsigned *batch, Text *body) { + Reader r = {.wire = w}; + char line[1024]; + if (!read_line(&r, line, sizeof(line)) || strncmp(line, "HTTP/1.", 7) || strlen(line) < 12) return false; + *status = atoi(line + 9); + *keep = line[7] == '1'; + *refused = false; + uint64_t length = 0; + bool sized = false, chunked = false; + while (read_line(&r, line, sizeof(line))) { + if (!*line) { + if (chunked) { + for (;;) { + uint64_t chunk; + if (!read_line(&r, line, sizeof(line))) return false; + chunk = strtoull(line, NULL, 16); + if (!chunk) break; + if (!take(&r, chunk, body) || !read_line(&r, line, sizeof(line))) return false; + } + while (read_line(&r, line, sizeof(line)) && *line) {} + return true; + } + if (*status == 204 || *status == 304 || *status < 200) return true; + if (sized) return take(&r, length, body); + *keep = false; /* the body runs to the close */ + while (fill(&r)) { + if (body && !take(&r, r.end - r.at, body)) return false; + r.at = r.end; + } + return true; + } + char *value = strchr(line, ':'); + if (!value) return false; + *value++ = 0; + while (*value == ' ' || *value == '\t') ++value; + if (!strcasecmp(line, "Content-Length")) { length = strtoull(value, NULL, 10); sized = true; } + else if (!strcasecmp(line, "Transfer-Encoding")) chunked = !strcasecmp(value, "chunked"); + else if (!strcasecmp(line, "Connection")) *keep = strcasecmp(value, "close") != 0 && *keep; + else if (!strcasecmp(line, "X-Pure-Error")) *refused = refusal(value); + else if (!strcasecmp(line, "X-Pure-Batch")) *batch = (unsigned)strtoul(value, NULL, 10); + } + return false; +} + +static void write_u32(unsigned char *p, uint32_t v) { for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(v >> (8 * i)); } + +/* The answer (status, body) to the send p as a message of the answer input: + * fields from the body's JSON object by name, others skipped; a field the + * body lacks from the send's field of that name and type; a u32 field named + * http_status from the status. False when a field has none of these. */ +static bool form_answer(const PureWire *w, int status, const Text *body, const Pending *p, Answer *out) { + const PureModule *m = w->module; + const PureInputShape *in = &m->input[w->answer_input], *sent = &m->input[w->output]; + size_t table = 8 * (size_t)in->field_count, total = table + in->prefix + body->size + p->sent_size; + unsigned char *bytes = calloc(1, total ? total : 1); + if (!bytes) return false; + unsigned char *prefix = bytes + table, *text = prefix + in->prefix, *copy = text + body->size; + if (body->size) memcpy(text, body->bytes, body->size); + if (p->sent_size) memcpy(copy, p->sent, p->sent_size); + PureJsonMessage message = {.module = m, .shape = in, .payload = prefix, .table = bytes, .lenient = true}; + size_t at = pure_json_space(text, 0, body->size); + if (at < body->size && text[at] == '{' && pure_json_message(&message, text, body->size, &at)) { + free(bytes); + return false; + } + for (uint32_t f = 0; f < in->field_count; ++f) { + if (message.seen[f]) continue; + const PureField *field = &m->field[in->field_start + f]; + if (field->type == PURE_FIELD_U32 && !strcmp(field->name, "http_status")) { + write_u32(prefix + field->offset, (uint32_t)status); + continue; + } + /* The send's field of the same name and type, and where its bytes are. */ + size_t start = sent->prefix, length = 0; + const PureField *from = NULL; + for (uint32_t g = 0; g < sent->field_count && !from; ++g) { + const PureField *other = &m->field[sent->field_start + g]; + bool variable = other->type == PURE_FIELD_BYTES || other->type == PURE_FIELD_TEXT; + if (variable) length = other->offset == PURE_FIELD_REST ? p->sent_size - start : read_u32(p->sent + other->offset); + if (!strcmp(other->name, field->name) && other->type == field->type) from = other; + else if (variable) start += length; + } + bool variable = field->type == PURE_FIELD_BYTES || field->type == PURE_FIELD_TEXT; + if (!from || (variable && (start > p->sent_size || length > p->sent_size - start))) { + free(bytes); + return false; + } + if (field->type == PURE_FIELD_U32 || field->type == PURE_FIELD_NUMBER) + memcpy(prefix + field->offset, p->sent + from->offset, field->type == PURE_FIELD_U32 ? 4 : 8); + else { + write_u32(bytes + 8 * f, (uint32_t)(copy + start - prefix)); + write_u32(bytes + 8 * f + 4, (uint32_t)length); + } + } + *out = (Answer){p->seq, bytes, total, table}; + return true; +} + +/* One request for count sends from p (numbered in a row): one object, or + * an array of them for a receiver that takes batches. A final answer to a + * send whose answers are wired is made into *answer. */ +static int post(PureWire *w, const Pending *p, unsigned count, Answer *answer) { + size_t body = count > 1 ? count + 1 : 0; + for (unsigned i = 0; i < count; ++i) body += p[i].size; + char head[2048], batch[32] = ""; + if (count > 1) snprintf(batch, sizeof(batch), "X-Pure-Count: %u\r\n", count); + int size = snprintf(head, sizeof(head), + "POST %s HTTP/1.1\r\nHost: %s:%s\r\nContent-Type: application/json\r\nContent-Length: %zu\r\n" + "X-Pure-Source: %s\r\nX-Pure-Sequence: %llu\r\n%sIdempotency-Key: %s-%llu\r\n\r\n", + w->path, w->host, w->port, body, w->source, (unsigned long long)p->seq, batch, w->source, + (unsigned long long)p->seq); + /* A kept connection the peer closed is tried once more on a new one. */ + for (int attempt = 0; attempt < 2; ++attempt) { + bool fresh = w->fd < 0; + if (fresh) { + int fd = connect_to(w); + if (fd < 0) return AGAIN; + pthread_mutex_lock(&w->lock); + w->fd = fd; + bool stopping = w->stopping; + pthread_mutex_unlock(&w->lock); + if (stopping) return AGAIN; + SSL *ssl = w->tls ? start_tls(w, fd) : NULL; + if (w->tls && !ssl) { + drop_connection(w); + return TLS_FAILED; + } + pthread_mutex_lock(&w->lock); + w->ssl = ssl; + pthread_mutex_unlock(&w->lock); + } + int status; + bool refused, keep, sent = send_all(w, head, (size_t)size); + unsigned batch_size = 0; + Text body = {0}; + for (unsigned i = 0; sent && i < count; ++i) + sent = (!i || count == 1 || send_all(w, ",", 1)) && (i || count == 1 || send_all(w, "[", 1)) && + send_all(w, p[i].body, p[i].size); + if (sent && count > 1) sent = send_all(w, "]", 1); + if (!sent || !read_answer(w, &status, &refused, &keep, &batch_size, w->answer_input >= 0 ? &body : NULL)) { + free(body.bytes); + drop_connection(w); + if (fresh) return AGAIN; + continue; + } + if (!keep) drop_connection(w); + if (batch_size && w->answer_input < 0) w->batch = batch_size; + if (status == 408 || status == 429 || status >= 500) { + free(body.bytes); + return AGAIN; + } + if (w->answer_input >= 0 && status >= 200 && status < 500) { /* a final answer: the app hears it */ + bool formed = form_answer(w, status, &body, p, answer); + free(body.bytes); + return formed ? DELIVERED : ANSWER_UNREAD; + } + free(body.bytes); + if ((status >= 200 && status < 300) || (status >= 400 && status < 500 && refused)) return DELIVERED; + return REFUSED_BY_PEER; + } + return AGAIN; +} + +static void *deliver(void *context) { + PureWire *w = context; + long pause_ms = 0; + uint64_t reported = 0; + pthread_mutex_lock(&w->lock); + for (;;) { + while (!w->stopping && !w->count) pthread_cond_wait(&w->changed, &w->lock); + if (w->stopping) break; + /* As many as the receiver takes at once; their bodies stay put while + * the queue only grows behind them. */ + unsigned count = w->batch > 1 ? (unsigned)(w->count < w->batch ? w->count : w->batch) : 1; + Pending next[BATCH_MOST]; + if (count > BATCH_MOST) count = BATCH_MOST; + memcpy(next, w->queue + w->head, count * sizeof(*next)); + ++w->attempts; + pthread_mutex_unlock(&w->lock); + Answer answer = {0}; + int outcome = post(w, next, count, &answer); + pthread_mutex_lock(&w->lock); + if (outcome == DELIVERED && answer.bytes) { /* for the owner, who then counts it delivered */ + if (w->answer_head + w->answer_count == w->answer_capacity) { + if (w->answer_head) { + memmove(w->answers, w->answers + w->answer_head, w->answer_count * sizeof(*w->answers)); + w->answer_head = 0; + } else { + size_t capacity = w->answer_capacity ? 2 * w->answer_capacity : 16; + Answer *grown = realloc(w->answers, capacity * sizeof(*grown)); + if (!grown) { free(answer.bytes); outcome = AGAIN; } + else { w->answers = grown; w->answer_capacity = capacity; } + } + } + if (outcome == DELIVERED) w->answers[w->answer_head + w->answer_count++] = answer; + } + if (outcome == DELIVERED) { + for (unsigned i = 0; i < count; ++i) { + free(w->queue[w->head + i].body); + free(w->queue[w->head + i].sent); + } + w->head += count; + w->count -= count; + if (!answer.bytes) w->delivered = next[count - 1].seq; + w->deliveries += count; + pause_ms = 0; + pthread_mutex_unlock(&w->lock); + if (w->notify) w->notify(w->context); + pthread_mutex_lock(&w->lock); + continue; + } + if (outcome != AGAIN && reported != next[0].seq) { + fprintf(stderr, "{\"wire\":\"%s\",\"seq\":%llu,\"error\":\"%s\"}\n", w->source, + (unsigned long long)next[0].seq, outcome == TLS_FAILED ? "the receiver's certificate is not trusted; trying again" : + outcome == ANSWER_UNREAD ? "its answer does not make the answer input's message; trying again" : + "the receiver does not take it; trying again"); + reported = next[0].seq; + } + pause_ms = outcome != AGAIN ? LONGEST_PAUSE_MS : pause_ms ? 2 * pause_ms : FIRST_PAUSE_MS; + if (pause_ms > LONGEST_PAUSE_MS) pause_ms = LONGEST_PAUSE_MS; + struct timespec until; + clock_gettime(CLOCK_REALTIME, &until); + until.tv_sec += pause_ms / 1000; + until.tv_nsec += (pause_ms % 1000) * 1000000; + if (until.tv_nsec >= 1000000000) { ++until.tv_sec; until.tv_nsec -= 1000000000; } + while (!w->stopping && pthread_cond_timedwait(&w->changed, &w->lock, &until) == 0) {} + } + pthread_mutex_unlock(&w->lock); + return NULL; +} + +bool pure_wire_start(PureWire *w, void (*notify)(void *), void *context) { + w->notify = notify; + w->context = context; + w->started = pthread_create(&w->thread, NULL, deliver, w) == 0; + return w->started; +} + +void pure_wire_close(PureWire *w) { + if (!w) return; + pthread_mutex_lock(&w->lock); + w->stopping = true; + if (w->fd >= 0) shutdown(w->fd, SHUT_RDWR); /* a send or an answer waited for ends now */ + pthread_cond_broadcast(&w->changed); + pthread_mutex_unlock(&w->lock); + if (w->started) pthread_join(w->thread, NULL); + if (w->ssl) SSL_free(w->ssl); + if (w->fd >= 0) close(w->fd); + for (size_t i = 0; i < w->count; ++i) { + free(w->queue[w->head + i].body); + free(w->queue[w->head + i].sent); + } + for (size_t i = 0; i < w->answer_count; ++i) free(w->answers[w->answer_head + i].bytes); + free(w->answers); + free(w->taken.bytes); + free(w->queue); + pthread_cond_destroy(&w->changed); + pthread_mutex_destroy(&w->lock); + free(w); +} diff --git a/serve/deliver.h b/serve/deliver.h new file mode 100644 index 0000000..2d1da29 --- /dev/null +++ b/serve/deliver.h @@ -0,0 +1,60 @@ +#ifndef PURE_DELIVER_H +#define PURE_DELIVER_H +#include "module.h" + +/* Delivering an app's saved sends over HTTP (docs/research/083-delivery-over-http.md). + * A wire takes one output's sends in order: each goes as a JSON object of + * its fields (groups as objects inside) by POST to a URL, with the headers + * X-Pure-Source (the sender's name for the output), X-Pure-Sequence and + * Idempotency-Key, one at a time, again and again until it is delivered: + * a 2xx answer, or a Pure receiver's refusal (a 4xx with X-Pure-Error + * naming a fault: it heard the message and refused it). 408, 429, 5xx and + * no answer are tried again after a pause (0.1 s doubling to 10 s); another + * 4xx is logged and tried again every 10 s. Nothing is ever dropped: what is + * not delivered stays in the app's outbox, and a receiver applies a number + * once, so a send delivered twice (its answer lost) is harmless. */ +typedef struct PureWire PureWire; + +/* A wire for the output named output of m to url (http[s]://host[:port]/path), + * as source; NULL with why in error when it cannot be. An https receiver's + * certificate must be trusted (the system's CAs, or the file given to + * pure_wire_trust before the first https wire) and name its host. */ +void pure_wire_trust(const char *ca_file); +PureWire *pure_wire_open(const PureModule *m, const char *output, const char *url, const char *source, + char *error, size_t size); +uint32_t pure_wire_output(const PureWire *); +/* The owner's side: queue a saved send numbered after the last queued one + * (it is encoded now); false when it cannot be (a bytes field that is not + * UTF-8 has no JSON form), and the wire then stops at it. */ +bool pure_wire_queue(PureWire *, uint64_t seq, PureSpan payload); +uint64_t pure_wire_queued(const PureWire *); +/* The last number delivered; any thread. */ +uint64_t pure_wire_delivered(PureWire *); +/* Deliver on a thread of the wire's own, calling notify after each delivery. */ +bool pure_wire_start(PureWire *, void (*notify)(void *), void *context); +/* Stop (a delivery under way is finished or abandoned) and free. */ +void pure_wire_close(PureWire *); +/* Answers as events: the final answer to each send (a 2xx, or a 4xx but 408 + * and 429) becomes a message event of input (a message without a key), in + * the order sent: its fields come from the answer's JSON object by name + * (other members are skipped), a field the answer lacks from the send's + * field of that name and type (as a request's id), and a u32 field named + * http_status is the answer's status. An answer that makes no message is + * logged and the send tried again every 10 s. Answers go one per request. */ +bool pure_wire_reply(PureWire *, uint32_t input, char *error, size_t size); +/* The owner's side: the next answer in order, or false. Taking it counts + * its send delivered (the owner applies the answer, then trims); what out + * points to is the wire's until the next call. The answer is applied with + * pure_app_deliver from the wire's source with the send's number, so an + * answer that comes again after a restart is applied once. */ +typedef struct { + uint32_t input; + uint64_t seq; + PureSpan payload; + const uint32_t *placed; + const char *source; +} PureWireAnswer; +bool pure_wire_answer(PureWire *, PureWireAnswer *out); +/* Counts for a host's report. */ +void pure_wire_counts(PureWire *, uint64_t *delivered, uint64_t *attempts); +#endif diff --git a/serve/message-json.c b/serve/message-json.c new file mode 100644 index 0000000..66c812d --- /dev/null +++ b/serve/message-json.c @@ -0,0 +1,281 @@ +#include "message-json.h" +#include "module-text.h" +#include +#include +#include +#include + +static void write_u32(unsigned char *p, uint32_t v) { for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(v >> (8 * i)); } + +static bool decimal(const char *text, uint64_t maximum, uint64_t *value) { + if (!*text) return false; + uint64_t n = 0; + for (; *text; ++text) { + if (*text < '0' || *text > '9') return false; + unsigned digit = (unsigned)(*text - '0'); + if (n > (maximum - digit) / 10) return false; + n = n * 10 + digit; + } + *value = n; + return true; +} + +/* A JSON number's text: -?(0|[1-9][0-9]*)(.[0-9]+)?([eE][+-]?[0-9]+)?, its + * length (0 if there is none). An integer has neither part after it. */ +static size_t json_number(const unsigned char *p, size_t size, bool *integer) { + size_t i = 0; + *integer = true; + if (i < size && p[i] == '-') ++i; + if (i == size || p[i] < '0' || p[i] > '9') return 0; + if (p[i++] != '0') while (i < size && p[i] >= '0' && p[i] <= '9') ++i; + if (i < size && p[i] == '.') { + *integer = false; + if (++i == size || p[i] < '0' || p[i] > '9') return 0; + while (i < size && p[i] >= '0' && p[i] <= '9') ++i; + } + if (i < size && (p[i] == 'e' || p[i] == 'E')) { + *integer = false; + if (++i < size && (p[i] == '+' || p[i] == '-')) ++i; + if (i == size || p[i] < '0' || p[i] > '9') return 0; + while (i < size && p[i] >= '0' && p[i] <= '9') ++i; + } + return i; +} + +/* A fixed field's value from its text (a query value or a JSON number) into + * the prefix: a u32 in range, or a finite number. */ +bool pure_json_fixed(const PureField *field, const unsigned char *text, size_t size, unsigned char *prefix) { + bool integer; + if (!size || json_number(text, size, &integer) != size || size > 40) return false; + char digits[48]; + memcpy(digits, text, size); + digits[size] = 0; + if (field->type == PURE_FIELD_U32) { + uint64_t n; + if (!integer || !decimal(digits, UINT32_MAX, &n)) return false; + write_u32(prefix + field->offset, (uint32_t)n); + return true; + } + double value = strtod(digits, NULL); + if (!isfinite(value)) return false; + memcpy(prefix + field->offset, &value, 8); + return true; +} + +/* A JSON string's contents (after its opening quote) up to the closing one: + * its length, or 0 when it does not close or holds a control byte. */ +static size_t json_string_end(const unsigned char *p, size_t size) { + for (size_t i = 0; i < size; ++i) { + /* Eight bytes at a time while none is a quote, a backslash or a control byte. */ + const uint64_t ones = UINT64_C(0x0101010101010101), highs = UINT64_C(0x8080808080808080); + for (uint64_t w, q, s; size - i >= 8; i += 8) { + memcpy(&w, p + i, 8); + q = w ^ (ones * '"'); + s = w ^ (ones * '\\'); + if (((q - ones) & ~q & highs) | ((s - ones) & ~s & highs) | ((w - ones * 0x20) & ~w & highs)) break; + } + if (i == size) break; + if (p[i] == '"') return i + 1; + if (p[i] < 0x20) return 0; + if (p[i] == '\\' && ++i == size) return 0; + } + return 0; +} + +static int hex_digit(unsigned char c) { + return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; +} + +static bool json_code(const unsigned char *p, size_t left, uint32_t *code) { + if (left < 4) return false; + *code = 0; + for (int i = 0; i < 4; ++i) { + int d = hex_digit(p[i]); + if (d < 0) return false; + *code = *code << 4 | (uint32_t)d; + } + return true; +} + +/* Unescape a JSON string's contents into out (never longer); false for a + * bad escape or a lone surrogate. The result is checked as UTF-8 after. */ +static bool json_unescape(const unsigned char *in, size_t size, unsigned char *out, size_t *written) { + size_t o = 0; + for (size_t i = 0; i < size;) { + unsigned char b = in[i++]; + if (b != '\\') { out[o++] = b; continue; } + if (i == size) return false; + b = in[i++]; + const char *plain = strchr("\"\\/bfnrt", b); + if (b && plain) { out[o++] = (unsigned char)"\"\\/\b\f\n\r\t"[plain - "\"\\/bfnrt"]; continue; } + uint32_t code, low; + if (b != 'u' || !json_code(in + i, size - i, &code)) return false; + i += 4; + if (code >= 0xDC00 && code <= 0xDFFF) return false; + if (code >= 0xD800 && code <= 0xDBFF) { + if (size - i < 6 || in[i] != '\\' || in[i + 1] != 'u' || !json_code(in + i + 2, size - i - 2, &low) || + low < 0xDC00 || low > 0xDFFF) return false; + i += 6; + code = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); + } + if (code < 0x80) out[o++] = (unsigned char)code; + else if (code < 0x800) { out[o++] = (unsigned char)(0xC0 | code >> 6); out[o++] = (unsigned char)(0x80 | (code & 0x3F)); } + else if (code < 0x10000) { + out[o++] = (unsigned char)(0xE0 | code >> 12); + out[o++] = (unsigned char)(0x80 | (code >> 6 & 0x3F)); + out[o++] = (unsigned char)(0x80 | (code & 0x3F)); + } else { + out[o++] = (unsigned char)(0xF0 | code >> 18); + out[o++] = (unsigned char)(0x80 | (code >> 12 & 0x3F)); + out[o++] = (unsigned char)(0x80 | (code >> 6 & 0x3F)); + out[o++] = (unsigned char)(0x80 | (code & 0x3F)); + } + } + *written = o; + return true; +} + +size_t pure_json_space(const unsigned char *p, size_t at, size_t size) { + while (at < size && (p[at] == ' ' || p[at] == '\t' || p[at] == '\n' || p[at] == '\r')) ++at; + return at; +} + +/* Skip any JSON value at json[*at] (at most 64 deep): a member of an + * answer that names no field. */ +static bool skip_value(const unsigned char *json, size_t size, size_t *at, unsigned depth) { + if (*at == size || depth > 64) return false; + unsigned char b = json[*at]; + bool integer; + if (b == '"') { + size_t end = json_string_end(json + *at + 1, size - *at - 1); + if (!end) return false; + *at += 1 + end; + return true; + } + if (b == '{' || b == '[') { + unsigned char close = b == '{' ? '}' : ']'; + *at = pure_json_space(json, *at + 1, size); + for (bool first = true; *at < size && json[*at] != close; first = false) { + if (!first) { + if (json[(*at)++] != ',') return false; + *at = pure_json_space(json, *at, size); + } + if (close == '}') { + size_t key_end = *at < size && json[*at] == '"' ? json_string_end(json + *at + 1, size - *at - 1) : 0; + if (!key_end) return false; + *at = pure_json_space(json, *at + 1 + key_end, size); + if (*at == size || json[(*at)++] != ':') return false; + *at = pure_json_space(json, *at, size); + } + if (!skip_value(json, size, at, depth + 1)) return false; + *at = pure_json_space(json, *at, size); + } + if (*at == size) return false; + ++*at; + return true; + } + static const char *const words[] = {"true", "false", "null"}; + for (size_t w = 0; w < 3; ++w) + if (size - *at >= strlen(words[w]) && !memcmp(json + *at, words[w], strlen(words[w]))) { + *at += strlen(words[w]); + return true; + } + size_t length = json_number(json + *at, size - *at, &integer); + *at += length; + return length > 0; +} + +/* The field named path.key (key alone at the top), or the field count when + * none is; group says whether fields are named inside it instead. */ +static uint32_t field_named(const PureJsonMessage *b, const char *path, size_t path_size, const unsigned char *key, + size_t key_size, bool *group) { + const PureModule *m = b->module; + size_t size = path_size + !!path_size + key_size; + *group = false; + for (uint32_t f = 0; f < b->shape->field_count; ++f) { + const char *name = m->field[b->shape->field_start + f].name; + size_t length = strlen(name); + if (length < size || memcmp(name, path, path_size) || (path_size && name[path_size] != '.') || + memcmp(name + path_size + !!path_size, key, key_size)) continue; + if (length == size) return f; + if (name[size] == '.') *group = true; + } + return b->shape->field_count; +} + +/* An object's members from json[*at] (its opening brace), in the group named + * path: a field's value, or a group's object. Strings stay where they are + * (an escaped one is unescaped over itself, which never makes it longer). + * NULL, or the error to answer. */ +static const char *members(PureJsonMessage *b, unsigned char *json, size_t size, size_t *at, const char *path, size_t path_size, + unsigned depth) { + const PureModule *m = b->module; + if (*at == size || json[(*at)++] != '{') return "json"; + *at = pure_json_space(json, *at, size); + for (bool first = true; *at < size && json[*at] != '}'; first = false) { + if (!first) { + if (json[(*at)++] != ',') return "json"; + *at = pure_json_space(json, *at, size); + } + size_t key_end = *at < size && json[*at] == '"' ? json_string_end(json + *at + 1, size - *at - 1) : 0; + if (!key_end) return "json"; + const unsigned char *key = json + *at + 1; + size_t key_size = key_end - 1; + *at = pure_json_space(json, *at + 1 + key_end, size); + if (*at == size || json[(*at)++] != ':') return "json"; + *at = pure_json_space(json, *at, size); + bool group; + uint32_t f = field_named(b, path, path_size, key, key_size, &group); + if (f == b->shape->field_count && !group && b->lenient) { /* not the app's: skipped */ + if (!skip_value(json, size, at, depth)) return "json"; + *at = pure_json_space(json, *at, size); + continue; + } + if (*at < size && json[*at] == '{') { + if (!group) return f < b->shape->field_count ? "json_type" : "json_field"; + char inner[PURE_MODULE_NAME]; + size_t inner_size = path_size + !!path_size + key_size; + memcpy(inner, path, path_size); + if (path_size) inner[path_size] = '.'; + memcpy(inner + path_size + !!path_size, key, key_size); + const char *why = members(b, json, size, at, inner, inner_size, depth + 1); + if (why) return why; + *at = pure_json_space(json, *at, size); + continue; + } + if (f == b->shape->field_count) return group ? "json_type" : "json_field"; + if (b->seen[f] || (b->given && b->given[f])) return "json_field"; + b->seen[f] = true; + const PureField *field = &m->field[b->shape->field_start + f]; + bool variable = field->type == PURE_FIELD_BYTES || field->type == PURE_FIELD_TEXT, integer; + if (*at < size && json[*at] == '"') { + size_t end = json_string_end(json + *at + 1, size - *at - 1), written; + if (!end) return "json"; + if (!variable) return "json_type"; + unsigned char *value = json + *at + 1; + if (!memchr(value, '\\', end - 1)) written = end - 1; + else if (!json_unescape(value, end - 1, value, &written)) return "json"; + if (field->type == PURE_FIELD_TEXT && !pure_utf8_valid((PureSpan){value, written})) return "json"; + write_u32(b->table + 8 * f, (uint32_t)(value - b->payload)); + write_u32(b->table + 8 * f + 4, (uint32_t)written); + *at = pure_json_space(json, *at + 1 + end, size); + continue; + } + size_t length = json_number(json + *at, size - *at, &integer); + if (!length) { + static const char *const words[] = {"true", "false", "null"}; + for (size_t w = 0; w < 3 && !length; ++w) + if (size - *at >= strlen(words[w]) && !memcmp(json + *at, words[w], strlen(words[w]))) return "json_type"; + return *at < size && json[*at] == '[' ? "json_type" : "json"; + } + if (variable || !pure_json_fixed(field, json + *at, length, b->payload)) return "json_type"; + *at = pure_json_space(json, *at + length, size); + } + if (*at == size) return "json"; + ++*at; + return NULL; +} + +const char *pure_json_message(PureJsonMessage *message, unsigned char *json, size_t size, size_t *at) { + return members(message, json, size, at, "", 0, 0); +} diff --git a/serve/message-json.h b/serve/message-json.h new file mode 100644 index 0000000..61211d6 --- /dev/null +++ b/serve/message-json.h @@ -0,0 +1,26 @@ +#ifndef PURE_MESSAGE_JSON_H +#define PURE_MESSAGE_JSON_H +#include "module.h" + +/* A message's fields from a JSON object read in place (docs/research/082-json-in-place.md): + * a u32 or a number for a fixed field (put into the prefix), a string for + * bytes or text (left where it is: an escaped one is unescaped over itself, + * which never makes it longer), a group's fields as an object inside. */ +typedef struct { + const PureModule *module; + const PureInputShape *shape; + unsigned char *payload; /* the prefix; a string's start is counted from here */ + unsigned char *table; /* for each field, its start and size (u32le each) */ + const bool *given; /* fields given elsewhere (a request's query), or NULL */ + bool lenient; /* a member naming no field is skipped, value and all (an API's answer) */ + bool seen[PURE_MODULE_FIELDS]; +} PureJsonMessage; + +/* The object at json[*at] (its opening brace), *at moved past it: NULL, or + * what is wrong ("json", "json_field": unknown or given twice, "json_type"). */ +const char *pure_json_message(PureJsonMessage *, unsigned char *json, size_t size, size_t *at); +size_t pure_json_space(const unsigned char *json, size_t at, size_t size); +/* A fixed field's value from its text (a query value or a JSON number) into + * the prefix: a u32 in range, or a finite number. */ +bool pure_json_fixed(const PureField *field, const unsigned char *text, size_t size, unsigned char *prefix); +#endif diff --git a/serve/serve.c b/serve/serve.c new file mode 100644 index 0000000..2ea7adb --- /dev/null +++ b/serve/serve.c @@ -0,0 +1,1820 @@ +#define _GNU_SOURCE +/* pure-serve: the HTTP host for aup modules (docs/research/075). + * + * An app's views and inputs are its HTTP interface, with no routes to write: + * GET /views/NAME answers a view (a number or text as text/plain, a table as + * JSON rows, ?from=N&count=M for a page of them) and POST /inputs/NAME + * delivers an event (a text input's text is the body; a key input is + * pressed, down then up; a message takes its u32 fields from the query and + * its bytes field from the body, and answers what its handler replies, a + * field of the message). A failed event answers with its fault. + * + * A module with keyed messages named show and edit also serves the document + * contract of server.c (GET and PUT /objects/KEY, If-Match or If-None-Match: + * *, ETag = revision, X-Pure-Error): GET is its show message and PUT its edit + * message. HTTP/1.1 connections persist and may pipeline; an epoll thread + * accepts, reads and writes without blocking. Requests go through the + * runtime core (a work slot per connection, the payload budget, ingress + * owned by the core, write replies borrowed from the request). After each + * round of events every ready request is applied to the module with + * deferred saving, one save stores the burst, and only then are its + * requests answered: an answer is durable, and a failed save fails the + * whole burst, reads included. + * + * With --threads N each thread has its own listener on the same port + * (SO_REUSEPORT), epoll set and runtime domain; the kernel spreads + * connections over them. A module of keyed documents alone runs one app per + * thread: they share nothing but the store, whose commits take turns and + * whose revisions settle concurrent edits of a key. States have one owner: + * for a module with them, one more thread owns the app, and the connection + * threads hand it their ready requests and answer what it hands back. Its + * bursts take the requests of every thread, one save each. */ +#include "deliver.h" +#include "message-json.h" +#include "module-store.h" +#include "module-text.h" +#include "runtime.h" +#include "store.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { HEADER_LIMIT = 4096, EVENTS = 256, BATCH = 256 }; +typedef enum { CLOSED, READING_HEAD, READING_BODY, WAITING, WRITING } Phase; +typedef struct Host Host; + +/* What a request names: a document, a view (a table's rows from, count) or + * an input. Kept per work slot, so an accepted event runs even if its + * connection is gone before the burst. */ +typedef enum { ROUTE_NONE, ROUTE_OBJECT, ROUTE_VIEW, ROUTE_INPUT } RouteKind; +typedef struct { RouteKind kind; uint32_t target; size_t from, count; } Route; + +/* Allocated when accepted and reused after closing: memory follows the + * connections that are open, not the configured maximum. */ +typedef struct Connection { + Host *host; + struct Connection *next; /* on the spare or the closed list */ + size_t index; /* in the host's table */ + int fd; + Phase phase; + bool readable, writable, keep_alive, put, post; + uint64_t deadline; + char key[65]; /* the document's key, or the view's or input's name */ + RouteKind route; + /* An app input's message: its fixed fields from the query, which fields + * were given there, and whether the body is a JSON object of the rest. + * Its ingress is [u32le payload size][payload]; with JSON it is [u32le + * payload size | PLACED][u32le start and size of each field][payload: the + * prefix, then the JSON as it came], its strings read where they are. */ + unsigned char fields[8 * PURE_MODULE_FIELDS]; + size_t field_bytes; + bool given[PURE_MODULE_FIELDS], json; + uint32_t input; + unsigned char *ingress; + char source[PURE_SOURCE + 1]; /* a delivered message's sender (X-Pure-Source), or empty */ + uint64_t sequence; /* and its number (X-Pure-Sequence) */ + unsigned batch; /* how many numbered messages the body holds (X-Pure-Count) */ + int64_t expected; + size_t body_size, received, in_used; + unsigned char *body; + PureTicket ticket; + struct iovec *out, small[4]; /* the answer: its head, then its body's extents */ + size_t out_count, out_at; + char head[320]; + char in[HEADER_LIMIT]; /* the header being read, then pipelined bytes */ +} Connection; + +/* One request applied in a burst, for the connection thread that answers + * it. A read answered from the store lends its extents: they point into a + * store mapping, which the resource holds and the runtime charges and + * releases with the ticket. */ +typedef struct { + Host *host; + PureTicket ticket; + PureInput input; + PureResult status; + int64_t revision; + PureResource resource; + PureSpan reply; + struct iovec *extents; + size_t extent_count; + char key[65]; + RouteKind route; + PureFault fault; /* a failed event's */ + const char *type; /* a view's content type */ +} Applied; + +/* The app and its store. A connection thread runs it itself (an app of + * documents only), or one owner thread runs it for all connection threads. + * The owner publishes a snapshot of the states after each save, and the + * connection threads show views from it themselves: reads never wait for + * the owner, which may be applying events or waiting for a sync. */ +typedef struct { + const PureModule *module; + PureStore *store; + PureApp app; + uint32_t show, edit; + uint64_t bursts, burst_events; + Applied *applied; /* a round's requests, grown to fit */ + size_t applied_capacity; + bool wrote; /* the last round saved an event or an edit */ + bool reads_first; /* the owner's: a round's reads before its writes */ + double next[PURE_MODULE_INPUTS]; /* when each timer ticks next (wall clock, ms) */ + uint64_t ticks; + /* The owner's: the last save's snapshot (NULL: the owner shows views), + * taken under the lock, and the app's changes it shows. */ + bool publishes; + pthread_mutex_t published_lock; + PureSnapshot *published; + uint64_t published_changes, publications; + /* The owner's: outputs wired to HTTP receivers, and how far each was trimmed. */ + PureWire *wires[PURE_MODULE_INPUTS]; + unsigned wire_count; + uint64_t trimmed[PURE_MODULE_INPUTS], answers; + bool held[PURE_MODULE_INPUTS]; /* an answer failed to apply: the wire's sends stay until a restart */ +} Executor; + +/* A request made ready: its effect, what it names, and the thread that answers it. */ +typedef struct { Host *host; PureEffect effect; Route route; } Job; + +/* The owner of an app with states served by several connection threads: + * their requests wait here, and it runs them in bursts, one save each + * however many threads they came from, handing each thread back its + * applied requests to answer. */ +typedef struct { + Executor executor; + pthread_mutex_t lock; + pthread_cond_t waiting; + Job *jobs; + size_t count, capacity; + uint64_t arrivals; /* requests handed to the owner, ever */ + bool stopping; + bool delivered; /* a wire delivered a send: trim it */ + Host *hosts; + unsigned host_count; +} Owner; + +struct Host { + const PureModule *module; + bool documents; /* the module serves /objects: it has show and edit */ + Executor local; /* the app, when this thread runs it */ + Owner *owner; /* or the owner that runs it */ + PureApp reader; /* shows the owner's snapshot here */ + Applied *views; /* a round's views shown here, answered together */ + size_t views_capacity; + uint64_t shown; /* views shown from snapshots */ + Route *routes; /* by work slot */ + PureRuntime runtime; + PureWork *work; + Connection **table; /* open connections by index */ + Connection **by_slot; /* the connection waiting on a work slot */ + Connection *spare, *closed; + Job *jobs; /* a round's requests: at most one per work slot */ + size_t capacity, active; + int epoll, listener; + uint64_t timeout_ms; + uint64_t accepted, requests, expired; + /* What the owner hands back to answer (swapped for answering), and the + * eventfd that says so; pending counts requests at the owner. */ + pthread_mutex_t lock; + Applied *done, *answering; + size_t done_count, done_capacity, answering_capacity, pending; + int wake; +}; + +/* Grow an array to hold `need` items; false when memory runs out. */ +static bool grow(void *array, size_t *capacity, size_t need, size_t size) { + if (need <= *capacity) return true; + size_t next = *capacity ? *capacity : 64; + while (next < need) next *= 2; + void *grown = realloc(*(void **)array, next * size); + if (!grown) return false; + *(void **)array = grown; + *capacity = next; + return true; +} + +static atomic_bool stopping; /* lock-free, so a signal handler may set it */ +static void stop(int signal_number) { (void)signal_number; atomic_store(&stopping, true); } + +static uint64_t now_ms(void) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (uint64_t)t.tv_sec * 1000 + (uint64_t)t.tv_nsec / 1000000; +} + +/* The time events see: the wall clock, in milliseconds since 1970 (UTC). */ +static double wall_ms(void) { + struct timespec t; + clock_gettime(CLOCK_REALTIME, &t); + return floor((double)t.tv_sec * 1000 + (double)t.tv_nsec / 1e6); +} + +static bool number(const char *text, uint64_t maximum, uint64_t *value) { + if (!*text) return false; + uint64_t n = 0; + for (; *text; ++text) { + if (*text < '0' || *text > '9') return false; + unsigned digit = (unsigned)(*text - '0'); + if (n > (maximum - digit) / 10) return false; + n = n * 10 + digit; + } + *value = n; + return true; +} + +/* ---- Connections ------------------------------------------------------ */ + +static void release_ticket(void *owner) { + Connection *c = owner; + if (c->ticket.runtime && !pure_runtime_release(c->ticket)) abort(); + c->ticket = (PureTicket){0}; + c->body = NULL; +} + +/* Everything an answer borrowed goes back with its ticket: the request's + * payload, a copied reply or the lent store extents. */ +static void finish_output(Connection *c) { + release_ticket(c); + if (c->out != c->small) free(c->out); + c->out = NULL; + c->out_count = c->out_at = 0; +} + +/* Events of this round may still name c: it is reused after the round. */ +static void close_connection(Connection *c) { + if (c->phase == CLOSED) return; + Host *host = c->host; + finish_output(c); + close(c->fd); /* also leaves the epoll set */ + c->phase = CLOSED; + host->table[c->index] = NULL; + c->next = host->closed; + host->closed = c; + --host->active; +} + +/* Answer with a body of `count` extents (`size` bytes in all); the ticket + * keeps them alive. One writev sends as much as the socket takes. */ +static void respond_typed(Connection *c, int status, const char *reason, const char *error, int64_t revision, + const char *type, const struct iovec *extents, size_t count, size_t size) { + char tag[48] = ""; + if (revision > 0) snprintf(tag, sizeof(tag), "ETag: \"%" PRId64 "\"\r\n", revision); + /* A delivery's answer tells how many numbered messages one request may carry. */ + char batch[32] = ""; + if (*c->source) snprintf(batch, sizeof(batch), "X-Pure-Batch: %d\r\n", BATCH); + int length = snprintf(c->head, sizeof(c->head), + "HTTP/1.1 %d %s\r\nContent-Length: %zu\r\n%sContent-Type: %s\r\n" + "Cache-Control: no-store\r\nX-Pure-Error: %s\r\n%s%s\r\n", + status, reason, size, c->keep_alive ? "" : "Connection: close\r\n", type, error, tag, batch); + c->out = count < sizeof(c->small) / sizeof(*c->small) ? c->small : malloc((count + 1) * sizeof(*c->out)); + c->phase = WRITING; + if (!c->out) { c->keep_alive = false; c->out = c->small; count = 0; } /* cannot send it: the close tells */ + c->out[0] = (struct iovec){c->head, (size_t)length}; + if (count) memcpy(c->out + 1, extents, count * sizeof(*extents)); + c->out_count = count + 1; + c->out_at = 0; +} + +static void respond_extents(Connection *c, int status, const char *reason, const char *error, int64_t revision, + const struct iovec *extents, size_t count, size_t size) { + respond_typed(c, status, reason, error, revision, "application/octet-stream", extents, count, size); +} + +static void respond(Connection *c, int status, const char *reason, const char *error, int64_t revision, + PureSpan body) { + struct iovec extent = {(void *)body.data, body.size}; + respond_extents(c, status, reason, error, revision, &extent, body.size ? 1 : 0, body.size); +} + +/* A request refused before it has a ticket; a body it came with is not read. */ +static void reject(Connection *c, int status, const char *reason, const char *error) { + if (c->body_size) c->keep_alive = false; + respond(c, status, reason, error, 0, (PureSpan){0}); +} + +/* A request the host cannot read further: answer and close. */ +static void refuse(Connection *c, int status, const char *reason, const char *error) { + release_ticket(c); + c->keep_alive = false; + respond(c, status, reason, error, 0, (PureSpan){0}); +} + +static void answer(Connection *c, PureOutcome result) { + switch (result.status) { + case PURE_OK: respond(c, 200, "OK", "none", result.revision, result.body); return; + case PURE_CONFLICT: respond(c, 412, "Precondition Failed", "conflict", result.revision, (PureSpan){0}); return; + case PURE_NOT_FOUND: respond(c, 404, "Not Found", "not_found", 0, (PureSpan){0}); break; + case PURE_BUSY: respond(c, 503, "Service Unavailable", "store_busy", 0, (PureSpan){0}); break; + case PURE_BUDGET: respond(c, 503, "Service Unavailable", "payload_budget", 0, (PureSpan){0}); break; + case PURE_ALLOCATION: respond(c, 503, "Service Unavailable", "allocation", 0, (PureSpan){0}); break; + case PURE_CLOSED: respond(c, 503, "Service Unavailable", "stopped", 0, (PureSpan){0}); break; + case PURE_DENIED: respond(c, 403, "Forbidden", "denied", 0, (PureSpan){0}); break; + case PURE_LIMIT: respond(c, 413, "Content Too Large", "limit", 0, (PureSpan){0}); break; + case PURE_UNCERTAIN: respond(c, 500, "Internal Server Error", "durability_unknown", 0, (PureSpan){0}); break; + default: respond(c, 500, "Internal Server Error", "storage", 0, (PureSpan){0}); break; + } + /* A failed request carries no body: the payload can go now. */ + if (!result.body.size) release_ticket(c); +} + +static const char *const fault_names[] = {"none", "divide", "non_finite", "clamp", "limit", "not_found", "encoding", + "exists"}; + +/* An app route's answer: a view's value, or what an event replied. A failed + * event names its fault: 404 not_found, 409 exists, 422 for the others; + * a view without a value is 404 no_value. */ +static void answer_app(Connection *c, PureOutcome result, bool view, PureFault fault, const char *type) { + const char *named = fault != PURE_FAULT_NONE ? fault_names[fault] : NULL; + if (result.status == PURE_OK) { + struct iovec extent = {(void *)result.body.data, result.body.size}; + respond_typed(c, 200, "OK", "none", 0, type ? type : "application/octet-stream", &extent, + result.body.size ? 1 : 0, result.body.size); + } else if (result.status == PURE_NOT_FOUND) { + respond(c, 404, "Not Found", view ? "no_value" : named ? named : "not_found", 0, (PureSpan){0}); + } else if (result.status == PURE_CONFLICT) { + respond(c, 409, "Conflict", named ? named : "conflict", 0, (PureSpan){0}); + } else if (named && (result.status == PURE_LIMIT || result.status == PURE_INVALID)) { + respond(c, 422, "Unprocessable Content", named, 0, (PureSpan){0}); + } else if (result.status == PURE_INVALID) { + respond(c, 400, "Bad Request", "invalid", 0, (PureSpan){0}); /* such as a text that is not UTF-8 */ + } else { + answer(c, result); + return; + } + if (!result.body.size) release_ticket(c); +} + +/* Parse one request head of `size` bytes (through the blank line): the same + * bounded HTTP/1.1 subset as server.c, plus Connection, POST and the app + * routes. A query stays in head for start_request; an unknown path is + * answered 404 once its head is read. */ +static bool parse_head(Connection *c, char *head, size_t size, char **query) { + for (size_t i = 0; i < size; ++i) { + unsigned char b = (unsigned char)head[i]; + if ((b < 32 && b != '\r' && b != '\n' && b != '\t') || b == 127) return false; + } + head[size - 2] = 0; /* keep the last CRLF of the last header */ + char *line = head, *next = strstr(line, "\r\n"); + if (!next) return false; + *next = 0; + char *path = strchr(line, ' '); + if (!path) return false; + *path++ = 0; + c->put = !strcmp(line, "PUT"); + c->post = !strcmp(line, "POST"); + if (!c->put && !c->post && strcmp(line, "GET")) return false; + char *version = strchr(path, ' '); + if (!version) return false; + *version++ = 0; + if (strcmp(version, "HTTP/1.1")) return false; + if ((*query = strchr(path, '?'))) *(*query)++ = 0; + static const struct { const char *prefix; RouteKind kind; } routes[] = { + {"/objects/", ROUTE_OBJECT}, {"/views/", ROUTE_VIEW}, {"/inputs/", ROUTE_INPUT}}; + const char *key = ""; + c->route = ROUTE_NONE; + for (size_t i = 0; i < sizeof(routes) / sizeof(*routes) && c->route == ROUTE_NONE; ++i) + if (!strncmp(path, routes[i].prefix, strlen(routes[i].prefix))) { + c->route = routes[i].kind; + key = path + strlen(routes[i].prefix); + } + size_t key_size = strlen(key); + if (c->route != ROUTE_NONE && (!key_size || key_size > 64)) return false; + for (size_t i = 0; i < key_size; ++i) { + char b = key[i]; + if (!((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '-' || b == '_')) + return false; + } + memcpy(c->key, key, key_size + 1); + /* Each route has its method: documents GET and PUT, views GET, inputs POST. */ + if ((c->route == ROUTE_OBJECT && (c->post || *query)) || (c->route == ROUTE_VIEW && (c->put || c->post)) || + (c->route == ROUTE_INPUT && !c->post)) return false; + bool length_seen = false, condition_seen = false, host_seen = false; + c->source[0] = 0; + c->sequence = 0; + c->batch = 1; + bool counted = false; + c->body_size = 0; + c->json = false; + c->expected = -1; + c->keep_alive = true; + line = next + 2; + while (*line) { + next = strstr(line, "\r\n"); + if (!next) return false; + *next = 0; + char *value = strchr(line, ':'); + if (!value || value == line) return false; + *value++ = 0; + for (char *p = line; *p; ++p) + if (!((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || (*p >= '0' && *p <= '9') || *p == '-')) + return false; + while (*value == ' ' || *value == '\t') ++value; + char *tail = value + strlen(value); + while (tail > value && (tail[-1] == ' ' || tail[-1] == '\t')) *--tail = 0; + uint64_t n; + if (!strcasecmp(line, "Content-Length")) { + if (length_seen || !number(value, SIZE_MAX, &n)) return false; + length_seen = true; + c->body_size = (size_t)n; + } else if (!strcasecmp(line, "Host")) { + if (host_seen || !*value) return false; + host_seen = true; + } else if (!strcasecmp(line, "If-Match")) { + if (condition_seen || tail - value < 3 || *value != '"' || tail[-1] != '"') return false; + tail[-1] = 0; + if (!number(value + 1, INT64_MAX, &n) || !n) return false; + c->expected = (int64_t)n; + condition_seen = true; + } else if (!strcasecmp(line, "If-None-Match")) { + if (condition_seen || strcmp(value, "*")) return false; + c->expected = 0; + condition_seen = true; + } else if (!strcasecmp(line, "Content-Type")) { + c->json = !strncasecmp(value, "application/json", 16); + } else if (!strcasecmp(line, "Connection")) { + if (!strcasecmp(value, "close")) c->keep_alive = false; + else if (strcasecmp(value, "keep-alive")) return false; + } else if (!strcasecmp(line, "X-Pure-Source")) { + /* A message delivered from another app's output: applied once per number. */ + size_t length = strlen(value); + if (*c->source || !length || length > PURE_SOURCE) return false; + for (size_t i = 0; i < length; ++i) if (value[i] <= ' ' || value[i] > '~') return false; + memcpy(c->source, value, length + 1); + } else if (!strcasecmp(line, "X-Pure-Sequence")) { + if (c->sequence || !number(value, UINT64_MAX, &n) || !n) return false; + c->sequence = n; + } else if (!strcasecmp(line, "X-Pure-Count")) { + /* Numbered messages from X-Pure-Sequence on, as a JSON array. */ + if (counted || !number(value, BATCH, &n) || !n) return false; + c->batch = (unsigned)n; + counted = true; + } else if (!strcasecmp(line, "Transfer-Encoding") || !strcasecmp(line, "Expect")) return false; + line = next + 2; + } + if (!*c->source != !c->sequence || (*c->source && c->route != ROUTE_INPUT) || (counted && !*c->source) || + c->sequence > UINT64_MAX - c->batch) return false; + if (c->put) return host_seen && length_seen && condition_seen; + if (c->post) return host_seen && !condition_seen; /* no Content-Length: no body */ + return host_seen && !condition_seen && c->body_size == 0; +} + +static void put_u32(unsigned char *p, uint32_t v) { for (unsigned i = 0; i < 4; ++i) p[i] = (unsigned char)(v >> (8 * i)); } +static void put_u64(unsigned char *p, uint64_t v) { for (unsigned i = 0; i < 8; ++i) p[i] = (unsigned char)(v >> (8 * i)); } +static uint32_t get_u32(const unsigned char *p) { return p[0] | p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; } +static uint64_t get_u64(const unsigned char *p) { return get_u32(p) | (uint64_t)get_u32(p + 4) << 32; } + +/* What an app route names in the module, with a message's fixed fields + * taken from the query into c->fields. NULL, or the error to answer (404 + * for a name the module lacks, 400 otherwise). */ +static const char *resolve(const Host *host, Connection *c, char *query, Route *route, int *status) { + const PureModule *m = host->module; + *status = 404; + c->field_bytes = 0; + memset(c->given, 0, sizeof(c->given)); + if (route->kind == ROUTE_NONE || (route->kind == ROUTE_OBJECT && !host->documents)) return "route"; + if (route->kind == ROUTE_OBJECT) return NULL; + int found = route->kind == ROUTE_VIEW ? pure_module_view(m, c->key) : pure_module_input(m, c->key); + /* An output is the app's to send and a timer ticks by itself: neither is posted. */ + if (found >= 0 && route->kind == ROUTE_INPUT && + (m->input[found].kind == PURE_KIND_OUTPUT || m->input[found].kind == PURE_KIND_TIMER)) found = -1; + if (found < 0) return route->kind == ROUTE_VIEW ? "no_view" : "no_input"; + *status = 400; + route->target = (uint32_t)found; + route->from = 0; + route->count = SIZE_MAX; + const PureInputShape *shape = route->kind == ROUTE_INPUT ? &m->input[found] : NULL; + bool message = shape && shape->kind == PURE_KIND_MESSAGE; + if (message && shape->keyed) return "keyed"; + if (*c->source && !message) return "delivery"; + if (!host->owner) c->source[0] = 0; /* an app without states or outputs applies it as a message */ + if (shape) { + /* A body is a text input's text, a message's one variable field, or a + * JSON object of a message's fields (two variable fields need one). */ + unsigned variables = 0; + for (uint32_t f = 0; message && f < shape->field_count; ++f) + variables += m->field[shape->field_start + f].type == PURE_FIELD_BYTES || + m->field[shape->field_start + f].type == PURE_FIELD_TEXT; + if (c->json && !message) return "json"; + if (!c->json && (variables > 1 || (c->body_size && shape->kind == PURE_KIND_KEY) || + (c->body_size && message && !variables))) return variables > 1 ? "json" : "body"; + c->input = (uint32_t)found; + c->field_bytes = message ? shape->prefix : 0; + } + /* name=value pairs: a table's from and count, a message's fixed fields. */ + for (char *pair = query; pair && *pair;) { + char *end = strchr(pair, '&'), *value = strchr(pair, '='); + if (end) *end = 0; + if (!value || (end && value > end)) return "query"; + *value++ = 0; + if (route->kind == ROUTE_VIEW) { + uint64_t n; + bool from = !strcmp(pair, "from"); + if (!number(value, SIZE_MAX, &n) || m->view_type[found] != PURE_VIEW_TABLE || (!from && strcmp(pair, "count"))) + return "query"; + if (from) route->from = (size_t)n; + else route->count = (size_t)n; + } else { + uint32_t f = 0; + while (f < (message ? shape->field_count : 0) && strcmp(m->field[shape->field_start + f].name, pair)) ++f; + const PureField *field = message && f < shape->field_count ? &m->field[shape->field_start + f] : NULL; + if (!field || c->given[f] || (field->type != PURE_FIELD_U32 && field->type != PURE_FIELD_NUMBER) || + !pure_json_fixed(field, (const unsigned char *)value, strlen(value), c->fields)) return "query"; + c->given[f] = true; + } + pair = end ? end + 1 : NULL; + } + for (uint32_t f = 0; message && !c->json && f < shape->field_count; ++f) { + uint8_t type = m->field[shape->field_start + f].type; + if ((type == PURE_FIELD_U32 || type == PURE_FIELD_NUMBER) && !c->given[f]) return "missing_field"; + } + for (uint32_t f = 0; c->batch > 1 && f < shape->field_count; ++f) + if (c->given[f]) return "delivery"; /* a batch's fields are all in its objects */ + if (c->batch > 1 && !c->json) return "delivery"; + if (!*c->source) c->batch = 1; + return NULL; +} + +/* Parsing a JSON body in place: the payload it is in, the placed fields' + * table, and which fields it gave. */ +#define PLACED (UINT32_C(1) << 31) +/* After the size: a delivered message's number (u64le), how many follow it + * (u32le: a batch) and its source (u8 length and the bytes), in DELIVERY + * bytes before the tables; then a table for each message, a prefix for each + * and the JSON. */ +#define DELIVERED (UINT32_C(1) << 30) +enum { DELIVERY = 80 }; +/* An app input's JSON body becomes its message: an object with a member for + * each field the query did not give (a u32, a number, or a string for bytes + * and text), groups as objects inside it, and nothing else. Fixed fields go + * into the prefix; strings stay in the body, and the table says where. + * NULL, or the error to answer. */ +static const char *convert(Connection *c) { + const PureInputShape *shape = &c->host->module->input[c->input]; + uint32_t delivered = get_u32(c->ingress) & DELIVERED; + size_t table = 8 * (size_t)shape->field_count; + unsigned char *tables = c->ingress + 4 + (delivered ? DELIVERY : 0), *prefixes = tables + c->batch * table; + unsigned char *json = c->body; + size_t size = c->body_size, at = pure_json_space(json, 0, size); + /* A batch is an array of its messages' objects. */ + if (c->batch > 1) { + if (at == size || json[at++] != '[') return "json"; + at = pure_json_space(json, at, size); + } + for (unsigned i = 0; i < c->batch; ++i) { + if (i) { + if (at == size || json[at++] != ',') return "json"; + at = pure_json_space(json, at, size); + } + PureJsonMessage b = {.module = c->host->module, .shape = shape, .table = tables + i * table, + .payload = prefixes + i * c->field_bytes, .given = c->given}; + const char *why = pure_json_message(&b, json, size, &at); + if (why) return why; + at = pure_json_space(json, at, size); + for (uint32_t f = 0; f < shape->field_count; ++f) + if (!b.seen[f] && !c->given[f]) return "missing_field"; + } + if (c->batch > 1 && (at == size || json[at++] != ']')) return "json"; + if (pure_json_space(json, at, size) != size) return "json"; + put_u32(c->ingress, (uint32_t)(c->batch * c->field_bytes + c->body_size) | PLACED | delivered); + return NULL; +} + +/* A request's body is in: an app input's JSON becomes its payload, and the + * request is ready, or it is answered 400 and never runs. */ +static void body_done(Connection *c) { + const char *why = c->route == ROUTE_INPUT && c->json ? convert(c) : NULL; + if (why) { + release_ticket(c); /* never made ready: it never runs */ + respond(c, 400, "Bad Request", why, 0, (PureSpan){0}); + return; + } + if (!pure_runtime_ready(c->ticket)) abort(); + c->phase = WAITING; +} + +/* A whole head is in `in`: admit the request and take any body bytes that + * came with it; later bytes stay for the next request. */ +static void start_request(Connection *c, size_t head_size) { + Host *host = c->host; + char head[HEADER_LIMIT], *query = NULL; + memcpy(head, c->in, head_size); + size_t extra = c->in_used - head_size; + memmove(c->in, c->in + head_size, extra); + c->in_used = extra; + if (!parse_head(c, head, head_size, &query)) { refuse(c, 400, "Bad Request", "request"); return; } + if (c->body_size > PURE_OBJECT_LIMIT) { refuse(c, 413, "Content Too Large", "limit"); return; } + ++host->requests; + Route route = {.kind = c->route}; + int status; + const char *why = resolve(host, c, query, &route, &status); + if (why) { reject(c, status, status == 404 ? "Not Found" : "Bad Request", why); return; } + /* A document's PUT carries its body; an input's event its payload's size, + * the payload (the fixed fields, then the body), and room for its JSON. */ + bool write = route.kind == ROUTE_INPUT || c->put; + size_t table = route.kind == ROUTE_INPUT && c->json ? 8 * (size_t)host->module->input[c->input].field_count : 0; + size_t lead = 4 + (route.kind == ROUTE_INPUT && *c->source ? DELIVERY : 0); + /* A batch has a table and a prefix for each of its messages. */ + size_t input_size = route.kind == ROUTE_INPUT ? lead + c->batch * (table + c->field_bytes) + c->body_size : + write ? c->body_size : 0; + PureResult admitted = pure_runtime_accept(&host->runtime, write ? PURE_INPUT_WRITE : PURE_INPUT_READ, c->key, + route.kind == ROUTE_INPUT ? 0 : c->expected, input_size, &c->ticket); + if (admitted != PURE_OK) { + c->ticket = (PureTicket){0}; + /* An unread body would desynchronize the stream. */ + if (c->body_size) c->keep_alive = false; + answer(c, (PureOutcome){.status = admitted}); + return; + } + host->by_slot[c->ticket.slot] = c; + host->routes[c->ticket.slot] = route; + if (!write) { c->phase = WAITING; return; } + unsigned char *ingress = c->ingress = pure_runtime_ingress(c->ticket, NULL); /* NULL for no bytes */ + c->body = ingress; + if (route.kind == ROUTE_INPUT) { + put_u32(ingress, (uint32_t)(c->batch * c->field_bytes + c->body_size) | (lead > 4 ? DELIVERED : 0)); + if (lead > 4) { + size_t length = strlen(c->source); + put_u64(ingress + 4, c->sequence); + put_u32(ingress + 12, c->batch); + ingress[16] = (unsigned char)length; + memcpy(ingress + 17, c->source, length); + } + if (c->field_bytes) memcpy(ingress + lead + c->batch * table, c->fields, c->field_bytes); + c->body = ingress + lead + c->batch * (table + c->field_bytes); + } + size_t take = c->in_used < c->body_size ? c->in_used : c->body_size; + if (take) memcpy(c->body, c->in, take); + memmove(c->in, c->in + take, c->in_used - take); + c->in_used -= take; + c->received = take; + c->phase = READING_BODY; + if (c->received == c->body_size) body_done(c); +} + +static size_t head_end(const Connection *c) { + for (size_t i = 3; i < c->in_used; ++i) + if (!memcmp(c->in + i - 3, "\r\n\r\n", 4)) return i + 1; + return 0; +} + +/* Advance a connection as far as its socket allows without blocking. */ +static void drive(Connection *c) { + for (;;) { + if (c->phase == READING_HEAD) { + size_t end = head_end(c); + if (end) { start_request(c, end); continue; } + if (c->in_used == HEADER_LIMIT) { refuse(c, 431, "Request Header Fields Too Large", "headers"); continue; } + if (!c->readable) return; + ssize_t n = recv(c->fd, c->in + c->in_used, HEADER_LIMIT - c->in_used, 0); + if (n > 0) { c->in_used += (size_t)n; c->deadline = now_ms() + c->host->timeout_ms; continue; } + if (n < 0 && errno == EAGAIN) { c->readable = false; return; } + if (n < 0 && errno == EINTR) continue; + close_connection(c); + return; + } + if (c->phase == READING_BODY) { + if (!c->readable) return; + ssize_t n = recv(c->fd, c->body + c->received, c->body_size - c->received, 0); + if (n > 0) { + c->received += (size_t)n; + c->deadline = now_ms() + c->host->timeout_ms; + if (c->received == c->body_size) body_done(c); + continue; + } + if (n < 0 && errno == EAGAIN) { c->readable = false; return; } + if (n < 0 && errno == EINTR) continue; + close_connection(c); + return; + } + if (c->phase == WRITING) { + if (!c->writable) return; + size_t left = c->out_count - c->out_at; + ssize_t n = writev(c->fd, c->out + c->out_at, (int)(left < IOV_MAX ? left : IOV_MAX)); + if (n < 0 && errno == EAGAIN) { c->writable = false; return; } + if (n < 0 && errno == EINTR) continue; + if (n < 0) { close_connection(c); return; } + c->deadline = now_ms() + c->host->timeout_ms; /* a slow reader is fine while it reads */ + for (size_t sent = (size_t)n; c->out_at < c->out_count;) { + struct iovec *v = &c->out[c->out_at]; + if (sent < v->iov_len) { + v->iov_base = (char *)v->iov_base + sent; + v->iov_len -= sent; + break; + } + sent -= v->iov_len; + ++c->out_at; + } + if (c->out_at < c->out_count) continue; + /* Answered: the next request may already be here. */ + finish_output(c); + c->deadline = now_ms() + c->host->timeout_ms; + if (!c->keep_alive) { close_connection(c); return; } + c->phase = READING_HEAD; + continue; + } + return; /* WAITING: the next burst answers */ + } +} + +/* ---- Bursts ----------------------------------------------------------- */ + +static void release_copy(void *allocation, size_t size) { + (void)size; + free(allocation); +} + +static void release_loan(void *allocation, size_t size) { + (void)size; + pure_mapping_release(allocation); +} + +static bool allocate_ingress(void *context, size_t size, PureResource *out) { + (void)context; + void *bytes = malloc(size ? size : 1); /* a key press or an empty text has no payload */ + if (!bytes) return false; + *out = (PureResource){.body = {bytes, size}, .allocation = bytes, .allocation_size = size, .release = release_copy}; + return true; +} + +typedef struct { unsigned char *bytes; size_t size; const unsigned char *first; size_t spans; } Reply; + +static bool measure_reply(void *context, PureSpan span) { + Reply *r = context; + if (!r->spans++) r->first = span.data; + r->size += span.size; + return true; +} + +/* A value's extents, if the store mapping holds every one. */ +typedef struct { struct iovec *extents; size_t count, capacity; const PureMapping *mapping; bool held; } Gather; + +static bool gather_extents(void *context, PureSpan span) { + Gather *g = context; + if (!span.size) return true; + g->held = g->held && pure_mapping_holds(g->mapping, span) && + grow(&g->extents, &g->capacity, g->count + 1, sizeof(*g->extents)); + if (g->held) g->extents[g->count++] = (struct iovec){(void *)span.data, span.size}; + return g->held; +} + +/* A gathered value goes out as the mapping's extents, not copied: the answer + * holds the mapping (charged to the budget) until it is sent and its ticket + * released. false (and nothing held) when some extent was not stored. */ +static bool lend(Applied *out, PureMapping *mapping, Gather *gather, bool read, size_t size) { + if (!read || !gather->held) { + free(gather->extents); + pure_mapping_release(mapping); + return false; + } + out->extents = gather->extents; + out->extent_count = gather->count; + out->resource = (PureResource){.body = {NULL, size}, .allocation = mapping, .allocation_size = size, + .release = release_loan}; + return true; +} + +static bool copy_reply(void *context, PureSpan span) { + Reply *r = context; + if (span.size) memcpy(r->bytes + r->size, span.data, span.size); + r->size += span.size; + return true; +} + +/* ---- App routes -------------------------------------------------------- */ + +typedef struct { unsigned char *bytes; size_t size, capacity; bool failed; } Output; + +static bool put_bytes(Output *o, const void *bytes, size_t size) { + if (o->failed) return false; + if (size > o->capacity - o->size) { + size_t capacity = o->capacity ? o->capacity : 256; + while (capacity - o->size < size) { + if (capacity > SIZE_MAX / 2) { o->failed = true; return false; } + capacity *= 2; + } + unsigned char *grown = realloc(o->bytes, capacity); + if (!grown) { o->failed = true; return false; } + o->bytes = grown; + o->capacity = capacity; + } + if (size) memcpy(o->bytes + o->size, bytes, size); + o->size += size; + return true; +} + +static bool put_span(void *context, PureSpan span) { return put_bytes(context, span.data, span.size); } + +/* A JSON string's contents: quotes, backslashes and control bytes escaped, + * other bytes (UTF-8, which Pure keeps valid) as they are. */ +static bool put_json_span(void *context, PureSpan span) { + Output *o = context; + const unsigned char *p = span.data; + size_t plain = 0; + for (size_t i = 0; i < span.size; ++i) { + unsigned char b = p[i]; + if (b >= 0x20 && b != '"' && b != '\\') continue; + char escape[8]; + int length = b == '"' || b == '\\' ? snprintf(escape, sizeof(escape), "\\%c", b) : + snprintf(escape, sizeof(escape), "\\u%04x", b); + put_bytes(o, p + plain, i - plain); + put_bytes(o, escape, (size_t)length); + plain = i + 1; + } + return put_bytes(o, p + plain, span.size - plain); +} + +static void put_number(Output *o, double value) { + char digits[32]; + put_bytes(o, digits, pure_format_number(value, digits)); +} + +/* A view's value: a number in Pure's shortest form or a text, as text; a + * table as {"rows":N,"from":F,"items":[{column: value}...]} for the rows + * asked, each cell computed only for them (a cell without a value is null). */ +static PureResult show_view(PureApp *app, const Route *route, size_t available, Applied *answer) { + const PureModule *m = app->module; + uint32_t view = route->target; + PureResource *out = &answer->resource; + const char **type = &answer->type; + /* Room for a page of short rows at once. */ + Output o = {0}; + PureResult status = PURE_OK; + double value; + if (m->view_type[view] == PURE_VIEW_TABLE && !(o.bytes = malloc(o.capacity = 1024))) return PURE_ALLOCATION; + if (m->view_type[view] == PURE_VIEW_NUM) { + *type = "text/plain; charset=utf-8"; + if (pure_app_view(app, view, &value)) put_number(&o, value); + else status = PURE_NOT_FOUND; + } else if (m->view_type[view] == PURE_VIEW_TEXT) { + *type = "text/plain; charset=utf-8"; + /* A text state shown as it is goes out as its stored extents. */ + int state = pure_module_text_state(m, view); + PureDraft *text = state >= 0 ? app->texts[state] : NULL; + PureVersion version = text ? pure_draft_version(text) : (PureVersion){0, 0}; + if (version.bytes > available) return PURE_BUDGET; + PureMapping *mapping = version.bytes ? pure_draft_mapping(text) : NULL; + if (mapping) { + Gather gather = {NULL, 0, 0, mapping, true}; + bool read = pure_draft_read_version(text, version, 0, (size_t)version.bytes, gather_extents, &gather) == PURE_OK; + if (lend(answer, mapping, &gather, read, (size_t)version.bytes)) return PURE_OK; + } + status = pure_app_view_text(app, view, put_span, &o); + } else { + *type = "application/json"; + size_t rows = pure_app_rows(app, view), from = route->from < rows ? route->from : rows; + size_t end = from + (route->count < rows - from ? route->count : rows - from); + /* Each column's key, ,"name": (the first row's first without the comma), made once. */ + struct { uint32_t view; size_t length; char key[PURE_MODULE_NAME + 4]; } columns[PURE_MODULE_VIEWS]; + unsigned count = 0; + for (uint32_t column = 0; column < m->views; ++column) + if (m->view_table[column] == view + 1) { + columns[count].view = column; + columns[count].length = (size_t)snprintf(columns[count].key, sizeof(columns[count].key), ",\"%s\":", + m->view_names[column]); + ++count; + } + char head[80]; + put_bytes(&o, head, (size_t)snprintf(head, sizeof(head), "{\"rows\":%zu,\"from\":%zu,\"items\":[", rows, from)); + for (size_t row = from; row < end && !o.failed; ++row) { + put_bytes(&o, ",{" + (row == from), 2 - (row == from)); + for (unsigned k = 0; k < count; ++k) { + put_bytes(&o, columns[k].key + !k, columns[k].length - !k); + if (m->view_type[columns[k].view] == PURE_VIEW_NUM) { + if (pure_app_cell(app, columns[k].view, row, &value)) put_number(&o, value); + else put_bytes(&o, "null", 4); + continue; + } + size_t mark = o.size; + put_bytes(&o, "\"", 1); + if (pure_app_cell_text(app, columns[k].view, row, put_json_span, &o) == PURE_OK) put_bytes(&o, "\"", 1); + else if (!o.failed) { + o.size = mark; + put_bytes(&o, "null", 4); + } + } + put_bytes(&o, "}", 1); + } + put_bytes(&o, "]}", 2); + } + if (status == PURE_OK && !o.bytes && !(o.bytes = malloc(1))) o.failed = true; /* an empty value has a buffer too */ + if (status == PURE_OK && o.failed) status = PURE_ALLOCATION; + if (status == PURE_OK && o.size > available) status = PURE_BUDGET; + if (status != PURE_OK) { + free(o.bytes); + return status; + } + *out = (PureResource){.body = {o.bytes, o.size}, .allocation = o.bytes, .allocation_size = o.size, + .release = release_copy}; + return PURE_OK; +} + +/* An input's event: a text, a key pressed (down, then up), or a message + * whose reply (a field of it) is borrowed from the request. */ +static PureResult apply_input(PureApp *app, const Route *route, const PureEffect *effect, Applied *out) { + const PureInputShape *shape = &app->module->input[route->target]; + uint32_t header = get_u32(effect->value.data); + const unsigned char *delivery = effect->value.data + 4; + unsigned batch = header & DELIVERED ? get_u32(delivery + 8) : 1; + size_t lead = 4 + (header & DELIVERED ? DELIVERY : 0), table = header & PLACED ? 8 * (size_t)shape->field_count : 0; + PureSpan payload = {effect->value.data + lead + batch * table, header & ~(PLACED | DELIVERED)}; + PureResult status; + if (shape->kind == PURE_KIND_TEXT) { + status = pure_app_text(app, route->target, payload); + } else if (shape->kind == PURE_KIND_KEY) { + status = pure_app_input(app, route->target, PURE_EDGE_DOWN); + if (status == PURE_OK) status = pure_app_input(app, route->target, PURE_EDGE_UP); + } else { + uint32_t placed[2 * PURE_MODULE_FIELDS]; + for (size_t i = 0; table && i < 2 * (size_t)shape->field_count; ++i) + placed[i] = get_u32(effect->value.data + lead + 4 * i); + if (header & DELIVERED) { + char source[PURE_SOURCE + 1]; + memcpy(source, delivery + 13, delivery[12]); + source[delivery[12]] = 0; + status = pure_app_deliver(app, route->target, source, get_u64(delivery), payload, table ? placed : NULL); + /* The rest of a batch, in order: an app's refusal was heard and the + * batch goes on; another failure stops it, to be sent again. */ + for (unsigned i = 1; i < batch && (status == PURE_OK || app->fault != PURE_FAULT_NONE); ++i) { + const unsigned char *at = effect->value.data + lead + i * table; + for (size_t k = 0; k < 2 * (size_t)shape->field_count; ++k) placed[k] = get_u32(at + 4 * k); + PureSpan next = {payload.data + i * shape->prefix, payload.size - i * shape->prefix}; + status = pure_app_deliver(app, route->target, source, get_u64(delivery) + i, next, placed); + } + if (batch > 1 && app->fault != PURE_FAULT_NONE) status = PURE_OK; + } else + status = table ? pure_app_message_fields(app, route->target, NULL, -1, payload, placed) : + pure_app_message(app, route->target, NULL, -1, payload); + Reply reply = {0}; + if (status == PURE_OK) pure_app_reply(app, measure_reply, &reply); + out->reply = (PureSpan){reply.size ? reply.first : NULL, reply.size}; + } + out->fault = app->fault; + return status; +} + +static void apply_effect(Executor *ex, const Job *job, Applied *out) { + PureApp *app = &ex->app; + const PureEffect *effect = &job->effect; + app->now = wall_ms(); + *out = (Applied){.host = job->host, .ticket = effect->ticket, .input = effect->input, .route = job->route.kind}; + memcpy(out->key, effect->key, strlen(effect->key) + 1); + if (job->route.kind == ROUTE_VIEW) { + out->status = show_view(app, &job->route, effect->available, out); + return; + } + if (job->route.kind == ROUTE_INPUT) { + out->status = apply_input(app, &job->route, effect, out); + if (out->status == PURE_OK) ++ex->burst_events; + return; + } + bool read = effect->input == PURE_INPUT_READ; + out->status = pure_app_message(app, read ? ex->show : ex->edit, effect->key, + read ? -1 : effect->expected, read ? (PureSpan){0} : effect->value); + out->revision = app->revision; + if (out->status != PURE_OK) return; + ++ex->burst_events; + Reply reply = {0}; + pure_app_reply(app, measure_reply, &reply); + if (!read) { + /* Checked at start: an edit replies a field of its request. */ + out->reply = (PureSpan){reply.size ? reply.first : NULL, reply.size}; + return; + } + if (reply.size > effect->available) { out->status = PURE_BUDGET; return; } + if (!reply.size) return; + /* A stored document is lent, extent by extent, not copied. */ + PureMapping *mapping = app->document ? pure_draft_mapping(app->document) : NULL; + if (mapping) { + Gather gather = {NULL, 0, 0, mapping, true}; + bool read = pure_app_reply(app, gather_extents, &gather) == PURE_OK; + if (lend(out, mapping, &gather, read, reply.size)) return; + } + unsigned char *copy = malloc(reply.size); + if (!copy) { out->status = PURE_ALLOCATION; return; } + reply = (Reply){.bytes = copy}; + pure_app_reply(app, copy_reply, &reply); + out->resource = (PureResource){.body = {copy, reply.size}, .allocation = copy, .allocation_size = reply.size, + .release = release_copy}; +} + +/* After a save, before its answers go: from now on views show what it + * saved, so a read made after a write's answer sees the write. A failed save + * returns the app to the last save's states, which the snapshot still shows. */ +static void publish(Executor *ex) { + if (!ex->publishes || (ex->published && ex->published_changes == ex->app.changes)) return; + PureSnapshot *made = pure_app_snapshot(&ex->app), *old; + pthread_mutex_lock(&ex->published_lock); + old = ex->published; + ex->published = made; + pthread_mutex_unlock(&ex->published_lock); + pure_snapshot_release(old); + ex->published_changes = ex->app.changes; + ++ex->publications; +} + +/* The saved sends of wired outputs not queued yet go to their wires, in + * order; a wire that cannot take one takes the rest at a later save. The + * outbox is looked at newest first, only as far back as the new sends go, + * however many older ones wait for their receivers. */ +typedef struct { uint32_t wire; uint64_t seq; PureSpan payload; } Fresh; +typedef struct { Executor *executor; Fresh *fresh; size_t count, capacity; unsigned done; bool failed; } Queueing; + +static bool find_fresh(void *context, uint32_t output, uint64_t seq, PureSpan payload) { + Queueing *q = context; + Executor *ex = q->executor; + for (unsigned i = 0; i < ex->wire_count; ++i) { + if (pure_wire_output(ex->wires[i]) != output || (q->done >> i & 1)) continue; + if (seq <= pure_wire_queued(ex->wires[i])) { + q->done |= 1u << i; /* older ones were queued before */ + break; + } + if (q->count == q->capacity) { + size_t capacity = q->capacity ? 2 * q->capacity : 64; + Fresh *grown = realloc(q->fresh, capacity * sizeof(*grown)); + if (!grown) { q->failed = true; return false; } + q->fresh = grown; + q->capacity = capacity; + } + q->fresh[q->count++] = (Fresh){i, seq, payload}; + break; + } + return q->done != (ex->wire_count >= 32 ? UINT32_MAX : (1u << ex->wire_count) - 1); +} + +static void after_save(Executor *ex) { + publish(ex); + if (!ex->wire_count) return; + Queueing q = {.executor = ex}; + pure_app_outbox_newest(&ex->app, find_fresh, &q); + bool blocked[PURE_MODULE_INPUTS] = {false}; + for (size_t k = q.failed ? 0 : q.count; k-- > 0;) { + const Fresh *f = &q.fresh[k]; + if (!blocked[f->wire]) blocked[f->wire] = !pure_wire_queue(ex->wires[f->wire], f->seq, f->payload); + } + free(q.fresh); +} + +/* The answers the wires made become events, in order; then what the wires + * delivered leaves the outbox (with the next save). True when an answer was + * applied (the caller saves). */ +static bool take_deliveries(Executor *ex) { + bool answered = false; + for (unsigned i = 0; i < ex->wire_count; ++i) { + PureWireAnswer answer; + while (pure_wire_answer(ex->wires[i], &answer)) { + ex->app.now = wall_ms(); + PureResult result = pure_app_deliver(&ex->app, answer.input, answer.source, answer.seq, answer.payload, + answer.placed); + if (result != PURE_OK && ex->app.fault == PURE_FAULT_NONE) { + ex->held[i] = true; /* not applied and not refused: sent again after a restart */ + fprintf(stderr, "{\"wire\":\"%s\",\"seq\":%" PRIu64 ",\"error\":\"its answer was not applied\"}\n", + answer.source, answer.seq); + } + pure_app_finish(&ex->app); + answered = true; + ++ex->answers; + } + uint64_t delivered = pure_wire_delivered(ex->wires[i]); + if (delivered <= ex->trimmed[i] || ex->held[i]) continue; + pure_app_trim(&ex->app, pure_wire_output(ex->wires[i]), delivered); + ex->trimmed[i] = delivered; + } + return answered; +} + +/* One save for a burst: its requests are durable with it or fail with it. */ +static void save_burst(Executor *ex, Applied *applied, size_t count) { + if (!count) return; + for (size_t i = 0; i < count; ++i) + ex->wrote = ex->wrote || (applied[i].status == PURE_OK && applied[i].input == PURE_INPUT_WRITE); + PureResult saved = pure_app_save(&ex->app); + ++ex->bursts; + if (saved == PURE_OK) after_save(ex); + for (size_t i = 0; i < count; ++i) + if (applied[i].status == PURE_OK) applied[i].status = saved; +} + +/* Run a round's requests in bursts. A burst ends before a document's key + * repeats or before its documents would outgrow one save, so every request + * is stored by its burst's save or fails with it; views and events of the + * app's states join any burst. Each group of answers goes to finish. + * + * The owner takes a round's reads first and answers them at once: a + * round's requests come from different connections (each has one request + * at a time), so nothing orders them, and placed before the writes the + * reads see only saved states (a round starts from a save). With writes + * going on this halves the reads' wait and doubles both rates. A single + * thread does not: it waits out each save itself, and reads answered first + * come back first, splitting its next writes over more saves (the owner + * gathers after a save instead). */ +static void execute(Executor *ex, const Job *jobs, size_t count, void (*finish)(void *, Applied *, size_t), + void *context) { + if (!count) return; + if (!grow(&ex->applied, &ex->applied_capacity, count, sizeof(*ex->applied))) abort(); + Applied *applied = ex->applied; + /* One look at the store per round: a request made after another shard's + * answer was sent is read after that answer's commit. */ + bool pinned = pure_store_pin(ex->store, true) == PURE_OK; + size_t reads = 0; + for (size_t i = 0; ex->reads_first && i < count; ++i) + if (jobs[i].effect.input == PURE_INPUT_READ) apply_effect(ex, &jobs[i], &applied[reads++]); + if (reads) finish(context, applied, reads); + size_t n = reads, start = reads, keys = 0; + for (size_t i = 0; i < count; ++i) { + if (ex->reads_first && jobs[i].effect.input == PURE_INPUT_READ) continue; + bool document = jobs[i].route.kind == ROUTE_OBJECT, repeated = false; + for (size_t k = start; k < n && document && !repeated; ++k) + repeated = applied[k].route == ROUTE_OBJECT && !strcmp(applied[k].key, jobs[i].effect.key); + if (repeated || (document && keys == PURE_APP_PENDING)) { + save_burst(ex, applied + start, n - start); + finish(context, applied + start, n - start); + start = n; + keys = 0; + } + apply_effect(ex, &jobs[i], &applied[n++]); + keys += document; + } + save_burst(ex, applied + start, n - start); + finish(context, applied + start, n - start); + if (pinned) pure_store_pin(ex->store, false); +} + +/* Tick every timer that is due, with one save for them (an app with states + * only: a timer can change nothing else). The milliseconds until the next + * is due, or -1 without timers. Missed ticks are not made up. */ +static long run_timers(Executor *ex) { + const PureModule *m = ex->module; + if (!m->states) return -1; + double now = wall_ms(), soonest = -1; + bool ticked = false; + for (uint32_t i = 0; i < m->inputs; ++i) { + if (m->input[i].kind != PURE_KIND_TIMER) continue; + if (!ex->next[i]) ex->next[i] = now + m->input[i].period; /* the first period starts now */ + if (ex->next[i] <= now) { + ex->app.now = now; + if (pure_app_tick(&ex->app, i) == PURE_OK) ++ex->ticks; + ticked = true; + ex->next[i] = now + m->input[i].period; + } + if (soonest < 0 || ex->next[i] - now < soonest) soonest = ex->next[i] - now; + } + if (ticked && pure_app_save(&ex->app) == PURE_OK) after_save(ex); /* a failed save undoes the ticks */ + return soonest < 0 ? -1 : (long)ceil(soonest); +} + +/* On the request's connection thread: complete its ticket and answer it if + * its connection still waits for it. */ +static void answer_applied(Host *host, Applied *a) { + if (!pure_runtime_complete(a->ticket, a->status, a->input == PURE_INPUT_READ ? &a->resource : NULL, + a->status == PURE_OK ? a->reply : (PureSpan){0}, a->revision)) abort(); + Connection *c = host->by_slot[a->ticket.slot]; + if (c && c->phase == WAITING && c->ticket.generation == a->ticket.generation) { + PureOutcome result; + if (!pure_runtime_result(a->ticket, &result)) abort(); + /* A lent answer's body is its extents (the runtime charged them). */ + if (result.status == PURE_OK && a->extents) + respond_typed(c, 200, "OK", "none", result.revision, a->type ? a->type : "application/octet-stream", + a->extents, a->extent_count, result.body.size); + else if (a->route != ROUTE_OBJECT) answer_app(c, result, a->route == ROUTE_VIEW, a->fault, a->type); + else answer(c, result); + drive(c); /* most answers fit the socket buffer at once */ + } + free(a->extents); +} + +static void answer_burst(void *context, Applied *applied, size_t count) { + (void)context; + for (size_t i = 0; i < count; ++i) answer_applied(applied[i].host, &applied[i]); +} + +/* On the owner: give each connection thread its applied requests, and wake it. */ +static void hand_back(void *context, Applied *applied, size_t count) { + Owner *owner = context; + for (unsigned h = 0; h < owner->host_count; ++h) { + Host *host = &owner->hosts[h]; + size_t mine = 0; + for (size_t i = 0; i < count; ++i) mine += applied[i].host == host; + if (!mine) continue; + pthread_mutex_lock(&host->lock); + if (!grow(&host->done, &host->done_capacity, host->done_count + mine, sizeof(*host->done))) abort(); + for (size_t i = 0; i < count; ++i) + if (applied[i].host == host) host->done[host->done_count++] = applied[i]; + pthread_mutex_unlock(&host->lock); + uint64_t one = 1; + if (write(host->wake, &one, sizeof(one)) != sizeof(one)) abort(); + } +} + +/* On a connection thread: answer what the owner handed back. */ +static void answer_handed(Host *host) { + uint64_t signals; + if (read(host->wake, &signals, sizeof(signals)) < 0 && errno != EAGAIN) abort(); + pthread_mutex_lock(&host->lock); + Applied *done = host->done; + size_t count = host->done_count, capacity = host->done_capacity; + host->done = host->answering; + host->done_capacity = host->answering_capacity; + host->done_count = 0; + pthread_mutex_unlock(&host->lock); + host->answering = done; + host->answering_capacity = capacity; + for (size_t i = 0; i < count; ++i) answer_applied(host, &done[i]); + host->pending -= count; +} + +/* The owner's newest snapshot, held, shown by this thread's reader; NULL + * when there is none. Taken for each round, after the round's answers from + * the owner: those were handed back after their saves' snapshots. */ +static PureSnapshot *read_published(Host *host) { + Executor *ex = &host->owner->executor; + pthread_mutex_lock(&ex->published_lock); + PureSnapshot *snapshot = ex->published; + if (snapshot) pure_snapshot_hold(snapshot); + pthread_mutex_unlock(&ex->published_lock); + if (snapshot) pure_app_read(&host->reader, snapshot); + return snapshot; +} + +/* Every request made ready since the last round: run here, or handed to the + * owner. Views are shown here from the snapshot when there is one, after the + * rest went to the owner, which meanwhile applies and saves them. Answering + * may make more ready; they form the next round. */ +static void serve_bursts(Host *host) { + for (;;) { + size_t count = 0; + PureEffect effect; + while (pure_runtime_take(&host->runtime, &effect) == PURE_TAKE_EFFECT) + host->jobs[count++] = (Job){host, effect, host->routes[effect.ticket.slot]}; + if (!count) return; + if (!host->owner) { + execute(&host->local, host->jobs, count, answer_burst, NULL); + continue; + } + PureSnapshot *snapshot = NULL; + size_t views = 0; + for (size_t i = 0; i < count; ++i) views += host->jobs[i].route.kind == ROUTE_VIEW; + if (views && !(snapshot = read_published(host))) views = 0; /* the owner shows them */ + size_t handed = count - views; + if (handed) { + Owner *owner = host->owner; + pthread_mutex_lock(&owner->lock); + if (!grow(&owner->jobs, &owner->capacity, owner->count + handed, sizeof(*owner->jobs))) abort(); + for (size_t i = 0; i < count; ++i) + if (!snapshot || host->jobs[i].route.kind != ROUTE_VIEW) owner->jobs[owner->count++] = host->jobs[i]; + owner->arrivals += handed; + pthread_cond_signal(&owner->waiting); + pthread_mutex_unlock(&owner->lock); + host->pending += handed; + } + if (!views) return; + /* Shown first and answered after, as a burst is. */ + if (!grow(&host->views, &host->views_capacity, views, sizeof(*host->views))) abort(); + size_t shown = 0; + for (size_t i = 0; i < count; ++i) { + const Job *job = &host->jobs[i]; + if (job->route.kind != ROUTE_VIEW) continue; + Applied *a = &host->views[shown++]; + *a = (Applied){.host = host, .ticket = job->effect.ticket, .input = job->effect.input, .route = ROUTE_VIEW}; + a->status = show_view(&host->reader, &job->route, job->effect.available, a); + } + pure_snapshot_release(snapshot); /* what answers lend holds its own mapping */ + for (size_t i = 0; i < shown; ++i) answer_applied(host, &host->views[i]); + host->shown += shown; + } +} + +/* The owner's thread: take every waiting request of every connection thread + * as one round, until they have all stopped. */ +static struct timespec later(struct timespec t, long nanoseconds) { + t.tv_sec += nanoseconds / 1000000000; + t.tv_nsec += nanoseconds % 1000000000; + if (t.tv_nsec >= 1000000000) { ++t.tv_sec; t.tv_nsec -= 1000000000; } + return t; +} + +static bool before(struct timespec a, struct timespec b) { + return a.tv_sec < b.tv_sec || (a.tv_sec == b.tv_sec && a.tv_nsec < b.tv_nsec); +} + +static void *run_owner(void *context) { + Owner *owner = context; + Job *spare = NULL; + size_t spare_capacity = 0; + bool gather = false; + uint64_t answered_at = 0; /* arrivals when the last round's answers were handed back */ + for (;;) { + pthread_mutex_lock(&owner->lock); + /* Wait for requests, or until the next timer is due; trim what the + * wires delivered meanwhile. */ + for (;;) { + bool delivered = owner->delivered; + owner->delivered = false; + pthread_mutex_unlock(&owner->lock); + long due = run_timers(&owner->executor); + bool answered = delivered && take_deliveries(&owner->executor); + pthread_mutex_lock(&owner->lock); + /* Answers are saved with the next round, or now if none waits. */ + if (answered && !owner->count && !owner->stopping) { + pthread_mutex_unlock(&owner->lock); + if (pure_app_save(&owner->executor.app) == PURE_OK) after_save(&owner->executor); + pthread_mutex_lock(&owner->lock); + } + if (owner->count || owner->stopping) break; + if (owner->delivered) continue; + if (due < 0) { + pthread_cond_wait(&owner->waiting, &owner->lock); + continue; + } + struct timespec wake; + clock_gettime(CLOCK_REALTIME, &wake); + wake = later(wake, due * 1000000L); + pthread_cond_timedwait(&owner->waiting, &owner->lock, &wake); + } + /* After a round that saved, its answers bring requests back from the + * connection threads: gather them until none has come for 100 µs (a + * millisecond at most), so they share one save as they would on a + * single thread, where answering takes that long anyway. The quiet + * counts only once a request came after the answers: requests that + * waited out the save alone would otherwise start the next round + * without the clients just answered, and the clients would settle + * into two halves taking turns, each save carrying half of them. */ + struct timespec now, end; + clock_gettime(CLOCK_REALTIME, &now); + end = later(now, 1000000); + while (gather && owner->count && !owner->stopping && before(now, end)) { + size_t seen = owner->count; + bool returned = owner->arrivals != answered_at; + struct timespec quiet = later(now, 100000); + pthread_cond_timedwait(&owner->waiting, &owner->lock, returned && before(quiet, end) ? &quiet : &end); + if (returned && owner->count == seen) break; + clock_gettime(CLOCK_REALTIME, &now); + } + if (!owner->count) { + pthread_mutex_unlock(&owner->lock); + break; + } + Job *jobs = owner->jobs; + size_t count = owner->count, capacity = owner->capacity; + owner->jobs = spare; + owner->capacity = spare_capacity; + owner->count = 0; + pthread_mutex_unlock(&owner->lock); + owner->executor.wrote = false; + execute(&owner->executor, jobs, count, hand_back, owner); + gather = owner->executor.wrote; + pthread_mutex_lock(&owner->lock); + answered_at = owner->arrivals; + pthread_mutex_unlock(&owner->lock); + spare = jobs; + spare_capacity = capacity; + } + free(spare); + return NULL; +} + +/* A write's reply is borrowed from its request, so the handler of an input + * served by PUT or POST may reply only a field of its message. */ +static bool replies_borrow(const PureModule *m, uint32_t input) { + for (uint32_t h = 0; h < m->handlers; ++h) { + if (m->handler[h].event != 1 + 2 * input) continue; + const PureBlock *code = &m->handler[h].code; + /* A field replied as it is, or a text field as its bytes. */ + for (uint32_t pc = code->start; pc < code->start + code->length; ++pc) { + if (m->code[pc].op != PURE_OP_REPLY) continue; + uint32_t from = pc > code->start && m->code[pc - 1].op == PURE_OP_BYTES_FROM ? pc - 1 : pc; + if (from == code->start || m->code[from - 1].op != PURE_OP_FIELD) return false; + } + } + return true; +} + +/* ---- Main ------------------------------------------------------------- */ + +static bool load_module(PureModule *m, const char *path, bool *documents, uint32_t *show_out, uint32_t *edit_out) { + char error[160]; + if (pure_module_read(path, m, error, sizeof(error)) != PURE_OK) { + fprintf(stderr, "Module: %s\n", error); + return false; + } + /* Documents are served when the module has show and edit, as keyed messages. */ + int show = pure_module_input(m, "show"), edit = pure_module_input(m, "edit"); + *documents = show >= 0 || edit >= 0; + if (*documents && (show < 0 || edit < 0 || m->input[show].kind != PURE_KIND_MESSAGE || !m->input[show].keyed || + m->input[edit].kind != PURE_KIND_MESSAGE || !m->input[edit].keyed)) { + fputs("Module: documents need keyed messages named show and edit\n", stderr); + return false; + } + for (uint32_t i = 0; i < m->inputs; ++i) { + bool served = (*documents && i == (uint32_t)edit) || (m->input[i].kind == PURE_KIND_MESSAGE && !m->input[i].keyed); + if (served && !replies_borrow(m, i)) { + fprintf(stderr, "Module: %s may reply only a field of its message\n", m->input_names[i]); + return false; + } + } + *show_out = (uint32_t)(show >= 0 ? show : 0); + *edit_out = (uint32_t)(edit >= 0 ? edit : 0); + return true; +} + +static void accept_all(Host *host) { + for (;;) { + int fd = accept4(host->listener, NULL, NULL, SOCK_NONBLOCK | SOCK_CLOEXEC); + if (fd < 0) return; + if (host->active == host->capacity) { close(fd); continue; } /* full: the peer sees the close */ + Connection *c = host->spare; + if (c) host->spare = c->next; + else if (!(c = malloc(sizeof(*c)))) { close(fd); continue; } + size_t index = 0; + while (host->table[index]) ++index; /* a free index exists: active < capacity */ + int one = 1; + setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + c->host = host; + c->index = index; + c->fd = fd; + c->phase = READING_HEAD; + c->readable = c->writable = c->keep_alive = true; + c->deadline = now_ms() + host->timeout_ms; + c->ticket = (PureTicket){0}; + c->out = NULL; + c->out_count = c->out_at = 0; + c->body = NULL; + c->in_used = 0; + struct epoll_event event = {.events = EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET, .data.ptr = c}; + if (epoll_ctl(host->epoll, EPOLL_CTL_ADD, fd, &event)) { + close(fd); + c->next = host->spare; + host->spare = c; + continue; + } + host->table[index] = c; + ++host->active; + ++host->accepted; + drive(c); + } +} + +static bool start_executor(Executor *ex, const PureModule *module, uint32_t show, uint32_t edit, const char *store) { + ex->module = module; + ex->show = show; + ex->edit = edit; + if (pure_store_open(store, &ex->store) != PURE_OK) { fputs("Cannot open the store\n", stderr); return false; } + if (pure_app_start(&ex->app, module, pure_store_data(ex->store)) != PURE_OK || !pure_app_defer(&ex->app, true)) { + fputs("Cannot start the module on the store\n", stderr); + return false; + } + return true; +} + +static void stop_executor(Executor *ex) { + pure_snapshot_release(ex->published); + pure_app_stop(&ex->app); + pure_store_close(ex->store); + free(ex->applied); +} + +/* One connection thread: its own listener on the shared port, epoll set, + * runtime domain and, unless an owner runs the app for all of them, app and + * store handle. */ +static bool start_shard(Host *host, const PureModule *module, bool documents, uint32_t show, uint32_t edit, + const char *store, size_t budget, unsigned *port) { + host->module = module; + host->documents = documents; + host->wake = -1; + if (!host->owner && !start_executor(&host->local, module, show, edit, store)) return false; + host->work = calloc(host->capacity, sizeof(*host->work)); + host->table = calloc(host->capacity, sizeof(*host->table)); + host->by_slot = calloc(host->capacity, sizeof(*host->by_slot)); + host->jobs = calloc(host->capacity, sizeof(*host->jobs)); + host->routes = calloc(host->capacity, sizeof(*host->routes)); + if (!host->work || !host->table || !host->by_slot || !host->jobs || !host->routes) { + fputs("Cannot allocate connections\n", stderr); + return false; + } + if (!pure_runtime_init(&host->runtime, host->work, host->capacity, host, (PureHost){.allocate = allocate_ingress}, + budget) || + !pure_runtime_enable_async(&host->runtime)) return false; + host->listener = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0); + int one = 1; + setsockopt(host->listener, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + setsockopt(host->listener, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)); + struct sockaddr_in address = {.sin_family = AF_INET, .sin_port = htons((uint16_t)*port), + .sin_addr.s_addr = htonl(INADDR_LOOPBACK)}; + socklen_t address_size = sizeof(address); + if (host->listener < 0 || bind(host->listener, (struct sockaddr *)&address, sizeof(address)) || + listen(host->listener, 4096) || getsockname(host->listener, (struct sockaddr *)&address, &address_size)) { + perror("listen"); + return false; + } + *port = ntohs(address.sin_port); /* the first shard picks it, the others join it */ + host->epoll = epoll_create1(EPOLL_CLOEXEC); + struct epoll_event listen_event = {.events = EPOLLIN, .data.ptr = NULL}; + if (host->epoll < 0 || epoll_ctl(host->epoll, EPOLL_CTL_ADD, host->listener, &listen_event)) { + perror("epoll"); + return false; + } + /* The owner says it handed requests back through an eventfd. */ + if (host->owner) { + struct epoll_event wake_event = {.events = EPOLLIN, .data.ptr = &host->wake}; + if (pthread_mutex_init(&host->lock, NULL) || (host->wake = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC)) < 0 || + epoll_ctl(host->epoll, EPOLL_CTL_ADD, host->wake, &wake_event)) { + perror("eventfd"); + return false; + } + } + return true; +} + +static void *run_shard(void *context) { + Host *host = context; + struct epoll_event events[EVENTS]; + uint64_t next_sweep = now_ms() + 1000; + while (!atomic_load(&stopping)) { + /* A thread that runs the app ticks its timers on time. */ + long due = host->owner ? -1 : run_timers(&host->local); + int ready = epoll_wait(host->epoll, events, EVENTS, due >= 0 && due < 1000 ? (int)due : 1000); + for (int e = 0; e < ready; ++e) { + Connection *c = events[e].data.ptr; + if (!c) { accept_all(host); continue; } + if (events[e].data.ptr == &host->wake) { answer_handed(host); continue; } + if (c->phase == CLOSED) continue; + if (events[e].events & EPOLLERR) { close_connection(c); continue; } + if (events[e].events & (EPOLLIN | EPOLLRDHUP | EPOLLHUP)) c->readable = true; + if (events[e].events & EPOLLOUT) c->writable = true; + drive(c); + } + serve_bursts(host); + uint64_t now = now_ms(); + if (now >= next_sweep) { + next_sweep = now + 1000; + for (size_t i = 0; i < host->capacity; ++i) { + Connection *c = host->table[i]; + if (c && c->phase != WAITING && now >= c->deadline) { ++host->expired; close_connection(c); } + } + } + /* The round is over: closed connections can be reused. */ + while (host->closed) { + Connection *c = host->closed; + host->closed = c->next; + c->next = host->spare; + host->spare = c; + } + } + pure_runtime_stop(&host->runtime); + serve_bursts(host); + /* What the owner still runs is answered before the connections close. */ + while (host->pending) { + struct pollfd wake = {host->wake, POLLIN, 0}; + if (poll(&wake, 1, -1) > 0) answer_handed(host); + } + for (size_t i = 0; i < host->capacity; ++i) if (host->table[i]) close_connection(host->table[i]); + return NULL; +} + +static void stop_shard(Host *host) { + if (!host->owner) stop_executor(&host->local); + pure_app_stop(&host->reader); + if (host->listener >= 0) close(host->listener); + if (host->epoll >= 0) close(host->epoll); + if (host->wake >= 0) close(host->wake); + while (host->closed) { + Connection *c = host->closed; + host->closed = c->next; + c->next = host->spare; + host->spare = c; + } + for (Connection *c = host->spare, *next; c; c = next) { next = c->next; free(c); } + free(host->work); + free(host->table); + free(host->by_slot); + free(host->jobs); + free(host->routes); + free(host->done); + free(host->answering); + free(host->views); +} + +/* A wire delivered a send: the owner trims it. */ +static void wire_delivered(void *context) { + Owner *owner = context; + pthread_mutex_lock(&owner->lock); + owner->delivered = true; + pthread_cond_signal(&owner->waiting); + pthread_mutex_unlock(&owner->lock); +} + +/* --send OUTPUT=URL: that output's saved sends go to url, from NAME.OUTPUT. */ +static bool open_wires(Executor *ex, const PureModule *m, const char *const *sends, unsigned count, const char *name) { + for (unsigned i = 0; i < count; ++i) { + const char *equals = strchr(sends[i], '='); + char output[PURE_MODULE_NAME], source[PURE_SOURCE + 2], error[160]; + size_t length = equals ? (size_t)(equals - sends[i]) : 0; + if (!length || length >= sizeof(output)) { + fprintf(stderr, "--send takes OUTPUT=URL, not %s\n", sends[i]); + return false; + } + memcpy(output, sends[i], length); + output[length] = 0; + int written = snprintf(source, sizeof(source), "%s.%s", name, output); + PureWire *wire = written > 0 && written <= PURE_SOURCE ? pure_wire_open(m, output, equals + 1, source, error, sizeof(error)) : NULL; + if (!wire) { + fprintf(stderr, "--send %s: %s\n", sends[i], written > PURE_SOURCE ? "the name and output are longer than 64 bytes" : error); + return false; + } + for (unsigned k = 0; k < ex->wire_count; ++k) + if (pure_wire_output(ex->wires[k]) == pure_wire_output(wire)) { + fprintf(stderr, "--send %s: the output is wired already\n", sends[i]); + pure_wire_close(wire); + return false; + } + ex->wires[ex->wire_count++] = wire; + } + return true; +} + +/* --reply OUTPUT=INPUT: the answers to that output's sends become INPUT's events. */ +static bool wire_replies(Executor *ex, const PureModule *m, const char *const *replies, unsigned count) { + for (unsigned i = 0; i < count; ++i) { + const char *equals = strchr(replies[i], '='); + char output[PURE_MODULE_NAME], error[160] = ""; + size_t length = equals ? (size_t)(equals - replies[i]) : 0; + int input = equals ? pure_module_input(m, equals + 1) : -1; + PureWire *wire = NULL; + if (length && length < sizeof(output)) { + memcpy(output, replies[i], length); + output[length] = 0; + int index = pure_module_input(m, output); + for (unsigned k = 0; k < ex->wire_count && index >= 0; ++k) + if (pure_wire_output(ex->wires[k]) == (uint32_t)index) wire = ex->wires[k]; + } + if (!wire || input < 0 || !pure_wire_reply(wire, (uint32_t)input, error, sizeof(error))) { + fprintf(stderr, "--reply %s: %s\n", replies[i], !wire ? "takes OUTPUT=INPUT of an output given --send" : + input < 0 ? "no such input" : error); + return false; + } + } + return true; +} + +/* --threads auto: a connection thread for each CPU this process may run on, + * less the one the owner of an app with states takes. */ +static unsigned auto_threads(bool owner) { + cpu_set_t set; + long cpus = sched_getaffinity(0, sizeof(set), &set) ? sysconf(_SC_NPROCESSORS_ONLN) : CPU_COUNT(&set); + if (cpus < 1) cpus = 1; + if (owner && cpus > 1) --cpus; + return cpus > 256 ? 256 : (unsigned)cpus; +} + +int main(int argc, char **argv) { + static PureModule module; + static Owner owner; + const char *module_path = NULL, *store = NULL, *name = NULL, *sends[PURE_MODULE_INPUTS], *replies[PURE_MODULE_INPUTS]; + unsigned send_count = 0, reply_count = 0; + unsigned port = 8767, threads = 1; + size_t budget = 64u << 20, capacity = 1024; + uint64_t timeout_ms = 10000; + for (int i = 1; i < argc; i += 2) { + uint64_t value; + if (i + 1 == argc) goto usage; + if (!strcmp(argv[i], "--module")) module_path = argv[i + 1]; + else if (!strcmp(argv[i], "--store")) store = argv[i + 1]; + else if (!strcmp(argv[i], "--send") && send_count < PURE_MODULE_INPUTS) sends[send_count++] = argv[i + 1]; + else if (!strcmp(argv[i], "--name")) name = argv[i + 1]; + else if (!strcmp(argv[i], "--send-ca")) pure_wire_trust(argv[i + 1]); + else if (!strcmp(argv[i], "--reply") && reply_count < PURE_MODULE_INPUTS) replies[reply_count++] = argv[i + 1]; + else if (!strcmp(argv[i], "--port") && number(argv[i + 1], 65535, &value)) port = (unsigned)value; + else if (!strcmp(argv[i], "--budget") && number(argv[i + 1], SIZE_MAX, &value) && value) budget = (size_t)value; + else if (!strcmp(argv[i], "--connections") && number(argv[i + 1], 1u << 20, &value) && value) capacity = (size_t)value; + else if (!strcmp(argv[i], "--threads") && !strcmp(argv[i + 1], "auto")) threads = 0; + else if (!strcmp(argv[i], "--threads") && number(argv[i + 1], 256, &value) && value) threads = (unsigned)value; + else if (!strcmp(argv[i], "--timeout-ms") && number(argv[i + 1], 3600000, &value) && value) timeout_ms = value; + else goto usage; + } + uint32_t show, edit; + bool documents; + if (!module_path || !store || !load_module(&module, module_path, &documents, &show, &edit)) goto usage; + if (!threads) threads = auto_threads(module.states); + if (pure_store_init(store) != PURE_OK) { fputs("Cannot initialize the store directory\n", stderr); return 1; } + signal(SIGPIPE, SIG_IGN); + struct sigaction stop_action = {.sa_handler = stop}; + sigemptyset(&stop_action.sa_mask); + sigaction(SIGINT, &stop_action, NULL); + sigaction(SIGTERM, &stop_action, NULL); + Host *hosts = calloc(threads, sizeof(*hosts)); + pthread_t *thread = calloc(threads, sizeof(*thread)), owner_thread; + if (!hosts || !thread) return 1; + /* States (and outputs, whose sends are the app's state) have one owner on + * a thread of its own, whose snapshots the connection threads read. */ + bool outputs = false; + for (uint32_t i = 0; i < module.inputs; ++i) outputs = outputs || module.input[i].kind == PURE_KIND_OUTPUT; + bool owned = module.states || outputs; + /* The sender's name: --name, or the module file's name without .pmod. */ + char stem[PURE_SOURCE + 1]; + if (!name) { + const char *base = strrchr(module_path, '/') ? strrchr(module_path, '/') + 1 : module_path; + size_t length = strlen(base); + if (length > 5 && !strcmp(base + length - 5, ".pmod")) length -= 5; + snprintf(stem, sizeof(stem), "%.*s", (int)length, base); + name = stem; + } + if (owned) { + if (!start_executor(&owner.executor, &module, show, edit, store) || pthread_mutex_init(&owner.lock, NULL) || + pthread_cond_init(&owner.waiting, NULL) || pthread_mutex_init(&owner.executor.published_lock, NULL)) + return 1; + owner.executor.reads_first = true; + owner.executor.publishes = true; + if (!open_wires(&owner.executor, &module, sends, send_count, name) || + !wire_replies(&owner.executor, &module, replies, reply_count)) return 2; + after_save(&owner.executor); /* the snapshot, and the sends stored before */ + for (unsigned i = 0; i < owner.executor.wire_count; ++i) + if (!pure_wire_start(owner.executor.wires[i], wire_delivered, &owner)) { fputs("Cannot start a wire\n", stderr); return 1; } + owner.hosts = hosts; + owner.host_count = threads; + } + for (unsigned i = 0; i < threads; ++i) { + hosts[i].capacity = capacity; + hosts[i].timeout_ms = timeout_ms; + hosts[i].listener = hosts[i].epoll = -1; + hosts[i].owner = owned ? &owner : NULL; + if (!start_shard(&hosts[i], &module, documents, show, edit, store, budget, &port)) return 1; + } + if (owned && pthread_create(&owner_thread, NULL, run_owner, &owner)) { fputs("Cannot start the owner\n", stderr); return 1; } + printf("{\"port\":%u,\"threads\":%u}\n", port, threads); + fflush(stdout); + for (unsigned i = 1; i < threads; ++i) + if (pthread_create(&thread[i], NULL, run_shard, &hosts[i])) { fputs("Cannot start a thread\n", stderr); return 1; } + run_shard(&hosts[0]); + for (unsigned i = 1; i < threads; ++i) pthread_join(thread[i], NULL); + if (owned) { + pthread_mutex_lock(&owner.lock); + owner.stopping = true; + pthread_cond_signal(&owner.waiting); + pthread_mutex_unlock(&owner.lock); + pthread_join(owner_thread, NULL); + } + size_t active = 0, retained = 0; + uint64_t accepted = 0, requests = 0, bursts = owner.executor.bursts, events = owner.executor.burst_events, expired = 0, + ticks = owner.executor.ticks, shown = 0; + for (unsigned i = 0; i < threads; ++i) { + shown += hosts[i].shown; + active += hosts[i].active; + retained += hosts[i].runtime.retained; + accepted += hosts[i].accepted; + requests += hosts[i].requests; + bursts += hosts[i].local.bursts; + events += hosts[i].local.burst_events; + ticks += hosts[i].local.ticks; + expired += hosts[i].expired; + stop_shard(&hosts[i]); + } + uint64_t delivered = 0, attempts = 0; + for (unsigned i = 0; i < owner.executor.wire_count; ++i) { + uint64_t d, a; + pure_wire_counts(owner.executor.wires[i], &d, &a); + delivered += d; + attempts += a; + pure_wire_close(owner.executor.wires[i]); /* what was not delivered stays in the outbox */ + } + if (owned) { + stop_executor(&owner.executor); + free(owner.jobs); + } + unsigned long groups, grouped; + pure_store_group_counts(&groups, &grouped); + fprintf(stderr, "{\"active\":%zu,\"retained_payload_bytes\":%zu,\"accepted\":%" PRIu64 ",\"requests\":%" PRIu64 + ",\"bursts\":%" PRIu64 ",\"burst_events\":%" PRIu64 ",\"ticks\":%" PRIu64 ",\"expired\":%" PRIu64 + ",\"threads\":%u,\"owner\":%s,\"snapshots\":%" PRIu64 ",\"views_from_snapshots\":%" PRIu64 + ",\"delivered\":%" PRIu64 ",\"delivery_attempts\":%" PRIu64 ",\"answers\":%" PRIu64 + ",\"commit_groups\":%lu,\"grouped_commits\":%lu}\n", + active, retained, accepted, requests, bursts, events, ticks, expired, threads, owned ? "true" : "false", + owner.executor.publications, shown, delivered, attempts, owner.executor.answers, groups, grouped); + free(hosts); + free(thread); + return 0; +usage: + fputs("pure-serve --module FILE.pmod --store DIRECTORY [--port 8767] [--threads 1|N|auto] [--budget BYTES] " + "[--connections 1024] [--timeout-ms 10000] [--send OUTPUT=http[s]://host:port/path ...] [--name NAME] " + "[--send-ca FILE] [--reply OUTPUT=INPUT ...]\n", stderr); + return 2; +} diff --git a/store/store-tree.c b/store/store-tree.c new file mode 100644 index 0000000..146e919 --- /dev/null +++ b/store/store-tree.c @@ -0,0 +1,1856 @@ +/* Selected storage experiment: every document lives in one append-only file as + * a persistent B-tree of byte extents. An edit writes only the leaves and the + * path it changes, so inserting or deleting in a large document costs O(log n) + * nodes instead of a whole-document copy. Unchanged extents are shared by old + * and new versions. The document catalog is the same structure holding fixed + * records sorted by key. A commit appends a record (its head: where its nodes + * are and their digest) and then its nodes, and syncs once: one contiguous + * range, which a flash device writes about twice as fast as two. Every few + * commits the head also goes into the older of two slots at the start, where + * a reader begins before following the records after it; readers take no + * lock and map one immutable prefix. */ +#define _POSIX_C_SOURCE 200809L +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif +#include "store.h" +#include "state.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { FREE = 0xffff }; +/* Each head slot has a block of its own: a torn write of one never touches + * the other. */ +enum { SLOT = 128, BLOCK = 4096, DATA = 2 * BLOCK, LEAF_MAX = 4096, LEAF_MIN = 1024, + FAN_MAX = 32, FAN_MIN = 8, RECORD = 96, KEY = 64, HEIGHT_MAX = 12, + BUFFER = 65536 }; +/* Written zeros kept after the published end (see prepare_tail). */ +enum { TAIL_MIN = 65536, TAIL_STEP = 262144 }; +#ifdef PURE_STORE_TEST +enum { COMPACT_SLACK = 65536 }; +#else +enum { COMPACT_SLACK = 1048576 }; +#endif +#define LEAF_BIT (UINT64_C(1) << 63) +#define MEMORY_BIT (UINT64_C(1) << 62) +#define LOCATION (MEMORY_BIT - 1) +/* Slots of this format, and of the one before tail records (still read). */ +static const unsigned char head_magic[8] = {'P','U','R','E','T','R','E','3'}; +static const unsigned char earlier_magic[8] = {'P','U','R','E','T','R','E','2'}; +static const unsigned char record_magic[8] = {'P','U','R','E','R','E','C','1'}; +/* Commits between two heads written to a slot. */ +enum { CHECKPOINT = 16 }; +static const unsigned char branch_magic[4] = {'P','T','B','1'}; +static const char tree_name[] = "pure.tree"; +/* Threads of one process take turns as the writer: the file lock arbitrates + * only between processes (it is the process's, and closing any descriptor of + * the lock file drops it). */ +static pthread_mutex_t writers = PTHREAD_MUTEX_INITIALIZER; +static uint64_t temporary_sequence; + +/* A reference is a file offset, or an index into this edit's node arena. + * bytes is the size of the whole subtree. Leaves hold raw document bytes. */ +typedef struct { uint64_t ref, bytes; } Entry; +typedef struct { Entry *items; size_t count, capacity; } List; +typedef struct { + unsigned char *bytes; /* leaf only */ + Entry *children; /* branch only */ + size_t size; /* leaf bytes, child count, or next free slot + 1 */ + unsigned height; /* FREE for a reclaimed slot */ +} Node; +/* One mapped version of the file, shared by the drafts of a store handle + * and by whoever still sends extents from it. */ +typedef struct PureMapping { const unsigned char *map; uint64_t end; unsigned refs; } Mapping; +typedef struct { + const unsigned char *map; + uint64_t end; + Mapping *shared; /* map belongs to this shared mapping, not to the tree */ + Node *nodes; + size_t count, capacity, free_head, free_count; /* free_head is slot + 1 */ + size_t bytes; /* payload held by arena nodes, including unreachable ones */ +} Tree; +/* A commit's bytes are start..end (none for a file synced whole before it was + * published); check is their digest. */ +typedef struct { uint64_t seq, end, catalog, catalog_bytes, live, start, check; } Head; +/* The xxHash64 construction without a seed: four multiply-rotate lanes over + * 32-byte stripes, several GB/s. An all-zero Digest is an empty one. */ +typedef struct { uint64_t lane[4], length; unsigned char held[32]; size_t count; } Digest; +typedef struct { + int fd; + uint64_t at; /* file offset of the next byte */ + size_t used; + bool failed; + Digest digest; /* of every byte written */ + unsigned char buffer[BUFFER]; +} Writer; +typedef struct { PureSpan part[3]; size_t index, offset; } Source; + +static bool fault(PureStoreStage stage) { +#ifdef PURE_STORE_TEST + if (pure_store_test_fault(stage)) { errno = EIO; return true; } +#else + (void)stage; +#endif + return false; +} + +static bool valid_key(const char *key) { + if (!key) return false; + size_t length = strlen(key); + if (!length || length > KEY) return false; + for (size_t i = 0; i < length; ++i) { + char b = key[i]; + if (!((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || + (b >= '0' && b <= '9') || b == '-' || b == '_')) return false; + } + return true; +} + +/* Runtime-owned values (app states) live under '.' followed by a valid key. + * Document keys never start with '.', so the two never meet. */ +static bool runtime_key(const char *key) { + return key && key[0] == '.' && strlen(key) <= KEY && valid_key(key + 1); +} + +static uint64_t decode64(const unsigned char *bytes) { + uint64_t value = 0; + for (unsigned i = 0; i < 8; ++i) value |= (uint64_t)bytes[i] << (8 * i); + return value; +} + +static void encode64(unsigned char *bytes, uint64_t value) { + for (unsigned i = 0; i < 8; ++i) bytes[i] = (unsigned char)(value >> (8 * i)); +} + +static uint64_t load64(const unsigned char *bytes) { + uint64_t value; + memcpy(&value, bytes, 8); +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + value = __builtin_bswap64(value); +#endif + return value; +} + +/* Digests find torn and missing writes; they are not a defence against a + * hostile file. */ +static const uint64_t P1 = UINT64_C(0x9E3779B185EBCA87), P2 = UINT64_C(0xC2B2AE3D27D4EB4F), + P3 = UINT64_C(0x165667B19E3779F9), P4 = UINT64_C(0x85EBCA77C2B2AE63), + P5 = UINT64_C(0x27D4EB2F165667C5); + +static uint64_t rotate(uint64_t value, unsigned bits) { return value << bits | value >> (64 - bits); } +static uint64_t mix(uint64_t lane, uint64_t word) { return rotate(lane + word * P2, 31) * P1; } + +static void stripes(Digest *digest, const unsigned char *bytes, size_t count) { + uint64_t a = digest->lane[0], b = digest->lane[1], c = digest->lane[2], d = digest->lane[3]; + for (; count; --count, bytes += 32) { + a = mix(a, load64(bytes)); + b = mix(b, load64(bytes + 8)); + c = mix(c, load64(bytes + 16)); + d = mix(d, load64(bytes + 24)); + } + digest->lane[0] = a, digest->lane[1] = b, digest->lane[2] = c, digest->lane[3] = d; +} + +static void digest_add(Digest *digest, const unsigned char *bytes, size_t length) { + digest->length += length; + if (digest->count) { + size_t take = 32 - digest->count < length ? 32 - digest->count : length; + memcpy(digest->held + digest->count, bytes, take); + digest->count += take; + bytes += take; + length -= take; + if (digest->count < 32) return; + stripes(digest, digest->held, 1); + digest->count = 0; + } + stripes(digest, bytes, length / 32); + digest->count = length % 32; + if (digest->count) memcpy(digest->held, bytes + length - digest->count, digest->count); +} + +static uint64_t digest_end(const Digest *digest) { + const uint64_t *lane = digest->lane; + uint64_t hash = rotate(lane[0], 1) + rotate(lane[1], 7) + rotate(lane[2], 12) + rotate(lane[3], 18); + for (unsigned i = 0; i < 4; ++i) hash = (hash ^ mix(0, lane[i])) * P1 + P4; + hash += digest->length; + size_t i = 0; + for (; i + 8 <= digest->count; i += 8) hash = rotate(hash ^ mix(0, load64(digest->held + i)), 27) * P1 + P4; + for (; i < digest->count; ++i) hash = rotate(hash ^ digest->held[i] * P5, 11) * P1; + hash ^= hash >> 33; + hash *= P2; + hash ^= hash >> 29; + hash *= P3; + return hash ^ hash >> 32; +} + +static uint64_t digest_of(const unsigned char *bytes, size_t length) { + Digest digest = {0}; + digest_add(&digest, bytes, length); + return digest_end(&digest); +} + +#ifdef PURE_STORE_TEST +static unsigned long syncs; +unsigned long pure_tree_test_syncs(void) { return __atomic_load_n(&syncs, __ATOMIC_RELAXED); } +#define COUNT_SYNC() __atomic_add_fetch(&syncs, 1, __ATOMIC_RELAXED) +#else +#define COUNT_SYNC() ((void)0) +#endif + +static bool sync_fd(int fd) { + int rc; + COUNT_SYNC(); + do { rc = fsync(fd); } while (rc && errno == EINTR); + return rc == 0; +} + +/* Data and the size needed to read it back, without timestamps. */ +static bool sync_data(int fd) { +#ifdef __APPLE__ + return sync_fd(fd); +#else + int rc; + COUNT_SYNC(); + do { rc = fdatasync(fd); } while (rc && errno == EINTR); + return rc == 0; +#endif +} + +static bool write_at(int fd, const unsigned char *bytes, size_t length, uint64_t offset) { + while (length) { + ssize_t n = pwrite(fd, bytes, length, (off_t)offset); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return false; + bytes += (size_t)n; + length -= (size_t)n; + offset += (uint64_t)n; + } + return true; +} + +static bool read_at(int fd, unsigned char *bytes, size_t length, uint64_t offset) { + while (length) { + ssize_t n = pread(fd, bytes, length, (off_t)offset); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return false; + bytes += (size_t)n; + length -= (size_t)n; + offset += (uint64_t)n; + } + return true; +} + +static void encode_head(unsigned char out[SLOT], const unsigned char magic[8], const Head *head) { + memset(out, 0, SLOT); + memcpy(out, magic, 8); + encode64(out + 8, head->seq); + encode64(out + 16, head->end); + encode64(out + 24, head->catalog); + encode64(out + 32, head->catalog_bytes); + encode64(out + 40, head->live); + encode64(out + 48, head->start); + encode64(out + 56, head->check); + encode64(out + SLOT - 8, digest_of(out, SLOT - 8)); +} + +/* A slot's head (record false) or a commit record's. */ +static bool decode_head(const unsigned char in[SLOT], bool record, Head *head) { + bool magic = record ? !memcmp(in, record_magic, 8) : !memcmp(in, head_magic, 8) || !memcmp(in, earlier_magic, 8); + if (!magic || decode64(in + SLOT - 8) != digest_of(in, SLOT - 8)) return false; + *head = (Head){decode64(in + 8), decode64(in + 16), decode64(in + 24), decode64(in + 32), + decode64(in + 40), decode64(in + 48), decode64(in + 56)}; + return head->seq && head->end <= (uint64_t)INT64_MAX && head->end <= SIZE_MAX && + head->start >= DATA && head->start <= head->end && head->catalog_bytes % RECORD == 0 && + !head->catalog == !head->catalog_bytes; +} + +/* ---- Trusted heads -------------------------------------------------------- + * One sync makes a commit's bytes and its head durable together, so a crash + * during it can leave the head on the device without all of the bytes: a + * head counts only once its digest matches them. The older slot needs no + * such luck. A writer builds only on a durable head, so the newer head was + * written after the older one's commit was complete on the device. + * The process remembers the heads it published or checked, so it reads a + * commit's bytes at most once, and which of them it made durable. */ +enum { UNKNOWN, BROKEN, CHECKED, DURABLE, PUBLISHING }; +enum { KNOWN = 32 }; +typedef struct { dev_t device; ino_t inode; uint64_t seq, check; int state; } Known; +/* The newest head that counts per file, to follow records from. */ +typedef struct { dev_t device; ino_t inode; Head head; } Newest; +enum { NEWEST = 4 }; +static struct { + pthread_mutex_t lock; + Known heads[KNOWN]; + unsigned next; + Known publishing; /* written here, not durable yet */ + Newest newest[NEWEST]; + unsigned next_newest; +} known = {.lock = PTHREAD_MUTEX_INITIALIZER}; + +static bool names(const Known *known_head, const struct stat *file, const Head *head) { + return known_head->state && known_head->device == file->st_dev && known_head->inode == file->st_ino && + known_head->seq == head->seq && known_head->check == head->check; +} + +static int recall(const struct stat *file, const Head *head) { + int state = UNKNOWN; + pthread_mutex_lock(&known.lock); + if (names(&known.publishing, file, head)) state = PUBLISHING; + for (unsigned i = 0; i < KNOWN && state == UNKNOWN; ++i) + if (names(&known.heads[i], file, head)) state = known.heads[i].state; + pthread_mutex_unlock(&known.lock); + return state; +} + +/* PUBLISHING marks the one head being made durable; any other state ends + * that mark (UNKNOWN only ends it). */ +static void remember(const struct stat *file, const Head *head, int state) { + Known entry = {file->st_dev, file->st_ino, head->seq, head->check, state}; + pthread_mutex_lock(&known.lock); + if (state == PUBLISHING) known.publishing = entry; + else if (names(&known.publishing, file, head)) known.publishing.state = UNKNOWN; + if (state != PUBLISHING && state != UNKNOWN) { + Known *at = NULL; + for (unsigned i = 0; i < KNOWN && !at; ++i) + if (names(&known.heads[i], file, head)) at = &known.heads[i]; + if (!at) known.heads[known.next++ % KNOWN] = entry; + else if (at->state != DURABLE) *at = entry; /* a durable head stays durable */ + } + if (state == CHECKED || state == DURABLE) { + Newest *at = NULL; + for (unsigned i = 0; i < NEWEST && !at; ++i) + if (known.newest[i].head.seq && known.newest[i].device == file->st_dev && + known.newest[i].inode == file->st_ino) at = &known.newest[i]; + if (!at) at = &known.newest[known.next_newest++ % NEWEST]; + else if (at->head.seq >= head->seq) at = NULL; + if (at) *at = (Newest){file->st_dev, file->st_ino, *head}; + } + pthread_mutex_unlock(&known.lock); +} + +static bool newest_known(const struct stat *file, Head *out) { + bool found = false; + pthread_mutex_lock(&known.lock); + for (unsigned i = 0; i < NEWEST && !found; ++i) + if (known.newest[i].head.seq && known.newest[i].device == file->st_dev && known.newest[i].inode == file->st_ino) { + *out = known.newest[i].head; + found = true; + } + pthread_mutex_unlock(&known.lock); + return found; +} + +#ifdef PURE_STORE_TEST +/* Test-only: forget every head, as a new process after a crash would. */ +void pure_tree_test_forget(void) { + pthread_mutex_lock(&known.lock); + memset(known.heads, 0, sizeof(known.heads)); + memset(known.newest, 0, sizeof(known.newest)); + known.publishing.state = UNKNOWN; + pthread_mutex_unlock(&known.lock); +} +#endif + +/* Whether a head's commit has all of its bytes. */ +static bool whole(int fd, const Head *head) { + unsigned char buffer[16384]; + Digest digest = {0}; + for (uint64_t at = head->start; at < head->end;) { + size_t n = head->end - at < sizeof(buffer) ? (size_t)(head->end - at) : sizeof(buffer); + if (!read_at(fd, buffer, n, at)) return false; + digest_add(&digest, buffer, n); + at += n; + } + return digest_end(&digest) == head->check; +} + +/* The newest head that counts. A reader passes over the head this process is + * still making durable, so the process never answers with a commit that a + * crash could take back; a writer never meets one (it is the only writer). */ +static bool pick(int fd, const struct stat *file, const unsigned char *slots[2], bool writer, Head *out, + uint64_t sequences[2]) { + Head heads[2]; + bool valid[2]; + for (unsigned i = 0; i < 2; ++i) { + valid[i] = decode_head(slots[i], false, &heads[i]) && heads[i].end <= (uint64_t)file->st_size; + if (sequences) sequences[i] = valid[i] ? heads[i].seq : 0; + } + unsigned newest = valid[1] && (!valid[0] || heads[1].seq > heads[0].seq); + for (unsigned turn = 0; turn < 2; ++turn) { + unsigned slot = turn ? !newest : newest; + const Head *head = &heads[slot]; + if (!valid[slot]) continue; + int state = recall(file, head); + if (state == BROKEN || (state == PUBLISHING && !writer)) continue; + if (state == UNKNOWN) { + bool counts = whole(fd, head); + remember(file, head, counts ? CHECKED : BROKEN); + if (!counts) continue; + } + *out = *head; + return true; + } + return false; +} + +/* The commit whose record follows head's bytes, if its own bytes are whole. */ +static bool follow(int fd, const struct stat *file, bool writer, const Head *head, Head *out) { + unsigned char record[SLOT]; + uint64_t size = (uint64_t)file->st_size; + Head next; + if (head->end > size || size - head->end < SLOT || !read_at(fd, record, SLOT, head->end) || + !decode_head(record, true, &next) || next.seq != head->seq + 1 || next.start != head->end + SLOT || + next.end > size) return false; + int state = recall(file, &next); + if (state == BROKEN || (state == PUBLISHING && !writer)) return false; + if (state == UNKNOWN) { + bool counts = whole(fd, &next); + remember(file, &next, counts ? CHECKED : BROKEN); + if (!counts) return false; + } + *out = next; + return true; +} + +/* Whether head's record is in the file as it was made. */ +static bool recorded(int fd, const struct stat *file, const Head *head) { + unsigned char stored[SLOT], made[SLOT]; + if (head->start < DATA + SLOT || head->end > (uint64_t)file->st_size) return false; + encode_head(made, record_magic, head); + return read_at(fd, stored, SLOT, head->start - SLOT) && !memcmp(stored, made, SLOT); +} + +/* The newest commit: from the newer slot's head that counts, or from the + * newest this process has seen in the file when its record is still there, + * along the records that follow. sequences gets the slots' (0: none). */ +static bool read_head(int fd, struct stat *file, bool writer, Head *out, uint64_t sequences[2]) { + unsigned char slots[BLOCK + SLOT]; + Head head, seen; + if (fstat(fd, file) || !S_ISREG(file->st_mode) || file->st_size < 0 || !read_at(fd, slots, sizeof(slots), 0) || + !pick(fd, file, (const unsigned char *[2]){slots, slots + BLOCK}, writer, &head, sequences)) return false; + if (newest_known(file, &seen) && seen.seq > head.seq && recorded(fd, file, &seen)) head = seen; + while (follow(fd, file, writer, &head, &head)) {} + *out = head; + return true; +} + +static bool map_tree(int fd, const struct stat *file, const Head *head, Tree *tree) { + *tree = (Tree){0}; + if ((uint64_t)file->st_size < head->end) return false; + void *map = mmap(NULL, (size_t)head->end, PROT_READ, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) return false; + *tree = (Tree){.map = map, .end = head->end}; + return true; +} + +/* A lent mapping may be released on another thread (a host that answers on + * its connection threads): its count is atomic. */ +static void release_mapping(Mapping *mapping) { + if (!mapping || __atomic_sub_fetch(&mapping->refs, 1, __ATOMIC_ACQ_REL)) return; + munmap((void *)mapping->map, (size_t)mapping->end); + free(mapping); +} + +static void release_tree(Tree *tree) { + if (tree->shared) release_mapping(tree->shared); + else if (tree->map) munmap((void *)tree->map, (size_t)tree->end); + for (size_t i = 0; i < tree->count; ++i) { + free(tree->nodes[i].bytes); + free(tree->nodes[i].children); + } + free(tree->nodes); + *tree = (Tree){0}; +} + +static const unsigned char *leaf_bytes(const Tree *tree, Entry entry) { + if (!(entry.ref & LEAF_BIT) || !entry.bytes || entry.bytes > LEAF_MAX) return NULL; + uint64_t at = entry.ref & LOCATION; + if (entry.ref & MEMORY_BIT) { + if (at >= tree->count) return NULL; + const Node *node = &tree->nodes[at]; + return node->bytes && node->size == entry.bytes ? node->bytes : NULL; + } + if (at < DATA || at > tree->end || entry.bytes > tree->end - at) return NULL; + return tree->map + at; +} + +/* Returns the child count of a checked branch at height, or 0. */ +static size_t children(const Tree *tree, Entry entry, unsigned height, Entry out[FAN_MAX]) { + if ((entry.ref & LEAF_BIT) || !height) return 0; + uint64_t at = entry.ref & LOCATION; + size_t count; + if (entry.ref & MEMORY_BIT) { + if (at >= tree->count) return 0; + const Node *node = &tree->nodes[at]; + if (node->bytes || node->height != height || !node->size || node->size > FAN_MAX) return 0; + count = node->size; + memcpy(out, node->children, count * sizeof(*out)); + } else { + if (at < DATA || at % 8 || at > tree->end || tree->end - at < 8) return 0; + const unsigned char *p = tree->map + at; + count = (size_t)p[4] | (size_t)p[5] << 8; + unsigned stored = (unsigned)p[6] | (unsigned)p[7] << 8; + if (memcmp(p, branch_magic, sizeof(branch_magic)) || stored != height || !count || + count > FAN_MAX || (tree->end - at - 8) / 16 < count) return 0; + for (size_t i = 0; i < count; ++i) + out[i] = (Entry){decode64(p + 8 + 16 * i), decode64(p + 16 + 16 * i)}; + } + uint64_t sum = 0; + for (size_t i = 0; i < count; ++i) { + bool leaf = (out[i].ref & LEAF_BIT) != 0; + if (!out[i].bytes || out[i].bytes > UINT64_MAX - sum || leaf != (height == 1)) return 0; + sum += out[i].bytes; + } + return sum == entry.bytes ? count : 0; +} + +static bool root_height(const Tree *tree, Entry root, unsigned *height) { + *height = 0; + if (!root.bytes) return !root.ref; + if (root.ref & LEAF_BIT) return true; + uint64_t at = root.ref & LOCATION; + if (root.ref & MEMORY_BIT) { + if (at >= tree->count || tree->nodes[at].bytes) return false; + *height = tree->nodes[at].height; + } else { + if (at < DATA || at > tree->end || tree->end - at < 8) return false; + *height = (unsigned)tree->map[at + 6] | (unsigned)tree->map[at + 7] << 8; + } + return *height >= 1 && *height <= HEIGHT_MAX; +} + +/* Visit a range as borrowed extents: 1 done, 0 damaged, -1 stopped by visit. */ +static int visit_range(const Tree *tree, Entry entry, unsigned height, uint64_t offset, + uint64_t length, bool (*visit)(void *, PureSpan), void *context) { + if (!height) { + const unsigned char *bytes = leaf_bytes(tree, entry); + if (!bytes || offset > entry.bytes || length > entry.bytes - offset) return 0; + return visit(context, (PureSpan){bytes + offset, (size_t)length}) ? 1 : -1; + } + Entry kids[FAN_MAX]; + size_t count = children(tree, entry, height, kids); + if (!count) return 0; + uint64_t start = 0; + for (size_t i = 0; i < count && length; ++i) { + uint64_t end = start + kids[i].bytes; + if (offset < end) { + uint64_t inside = offset - start, take = kids[i].bytes - inside; + if (take > length) take = length; + int result = visit_range(tree, kids[i], height - 1, inside, take, visit, context); + if (result != 1) return result; + offset += take; + length -= take; + } + start = end; + } + return length ? 0 : 1; +} + +static bool copy_span(void *context, PureSpan span) { + unsigned char **out = context; + memcpy(*out, span.data, span.size); + *out += span.size; + return true; +} + +static bool copy_range(const Tree *tree, Entry entry, unsigned height, uint64_t offset, + uint64_t length, unsigned char *out) { + return !length || visit_range(tree, entry, height, offset, length, copy_span, &out) == 1; +} + +/* The bytes of a range held by one leaf, or NULL when it spans leaves. */ +static const unsigned char *single_leaf(const Tree *tree, Entry entry, unsigned height, + uint64_t offset, uint64_t length) { + while (height) { + Entry kids[FAN_MAX]; + size_t count = children(tree, entry, height, kids), i = 0; + uint64_t start = 0; + if (!count) return NULL; + while (i + 1 < count && offset >= start + kids[i].bytes) start += kids[i++].bytes; + if (offset - start > kids[i].bytes || length > kids[i].bytes - (offset - start)) return NULL; + entry = kids[i]; + offset -= start; + --height; + } + const unsigned char *bytes = leaf_bytes(tree, entry); + return bytes && offset <= entry.bytes && length <= entry.bytes - offset ? bytes + offset : NULL; +} + +static bool push(List *list, Entry entry) { + if (list->count == list->capacity) { + size_t capacity = list->capacity ? 2 * list->capacity : 8; + Entry *items = realloc(list->items, capacity * sizeof(*items)); + if (!items) return false; + list->items = items; + list->capacity = capacity; + } + list->items[list->count++] = entry; + return true; +} + +static size_t node_bytes(const Node *node) { + return node->bytes ? node->size : node->size * sizeof(Entry); +} + +static bool add_node(Tree *tree, Node node, uint64_t *index) { + tree->bytes += node_bytes(&node); + if (tree->free_head) { + *index = tree->free_head - 1; + tree->free_head = tree->nodes[*index].size; + --tree->free_count; + tree->nodes[*index] = node; + return true; + } + if (tree->count == tree->capacity) { + size_t capacity = tree->capacity ? 2 * tree->capacity : 64; + Node *nodes = realloc(tree->nodes, capacity * sizeof(*nodes)); + if (!nodes) { tree->bytes -= node_bytes(&node); return false; } + tree->nodes = nodes; + tree->capacity = capacity; + } + tree->nodes[tree->count] = node; + *index = tree->count++; + return true; +} + +static void take(Source *source, unsigned char *out, size_t length) { + while (length) { + PureSpan part = source->part[source->index]; + size_t left = part.size - source->offset; + if (!left) { ++source->index; source->offset = 0; continue; } + size_t n = left < length ? left : length; + memcpy(out, part.data + source->offset, n); + out += n; + length -= n; + source->offset += n; + } +} + +/* Balanced leaves: every leaf of a multi-leaf result holds >= LEAF_MAX / 2. */ +static bool chunk_bytes(Tree *tree, Source *source, uint64_t total, List *out) { + uint64_t pieces = (total + LEAF_MAX - 1) / LEAF_MAX; + for (uint64_t i = 0; i < pieces; ++i) { + size_t size = (size_t)(total / pieces + (i < total % pieces)); + unsigned char *bytes = malloc(size); + uint64_t index; + if (!bytes) return false; + take(source, bytes, size); + if (!add_node(tree, (Node){.bytes = bytes, .size = size}, &index)) { free(bytes); return false; } + if (!push(out, (Entry){LEAF_BIT | MEMORY_BIT | index, size})) return false; + } + return true; +} + +static bool chunk_entries(Tree *tree, const Entry *items, size_t count, unsigned height, List *out) { + size_t pieces = (count + FAN_MAX - 1) / FAN_MAX; + for (size_t i = 0, at = 0; i < pieces; ++i) { + size_t size = count / pieces + (i < count % pieces); + Entry *kids = malloc(size * sizeof(*kids)); + uint64_t sum = 0, index; + if (!kids) return false; + memcpy(kids, items + at, size * sizeof(*kids)); + for (size_t k = 0; k < size; ++k) sum += kids[k].bytes; + if (!add_node(tree, (Node){.children = kids, .size = size, .height = height}, &index)) { + free(kids); + return false; + } + if (!push(out, (Entry){MEMORY_BIT | index, sum})) return false; + at += size; + } + return true; +} + +/* Only nodes created by this edit can be underfull; stored siblings are left as they are. */ +static bool underfull(const Tree *tree, Entry entry, unsigned height) { + if (!(entry.ref & MEMORY_BIT)) return false; + return height ? tree->nodes[entry.ref & LOCATION].size < FAN_MIN : entry.bytes < LEAF_MIN; +} + +static bool replace(List *list, size_t at, size_t removed, const List *with) { + size_t count = list->count - removed + with->count; + if (count > list->capacity) { + Entry *items = realloc(list->items, count * sizeof(*items)); + if (!items) return false; + list->items = items; + list->capacity = count; + } + memmove(list->items + at + with->count, list->items + at + removed, + (list->count - at - removed) * sizeof(*list->items)); + memcpy(list->items + at, with->items, with->count * sizeof(*list->items)); + list->count = count; + return true; +} + +static bool merge(Tree *tree, Entry a, Entry b, unsigned height, List *out); + +/* Merge each new underfull node with a neighbour. A merge removes a node or + * yields two nodes of at least half capacity, so the loop terminates. */ +static bool repair(Tree *tree, List *all, unsigned height) { + bool ok = true; + for (size_t i = 0; ok && all->count > 1 && i < all->count;) { + if (!underfull(tree, all->items[i], height)) { ++i; continue; } + size_t at = i + 1 < all->count ? i : i - 1; + List merged = {0}; + ok = merge(tree, all->items[at], all->items[at + 1], height, &merged) && + replace(all, at, 2, &merged); + free(merged.items); + i = at; + } + return ok; +} + +static bool merge(Tree *tree, Entry a, Entry b, unsigned height, List *out) { + if (!height) { + const unsigned char *x = leaf_bytes(tree, a), *y = leaf_bytes(tree, b); + if (!x || !y) return false; + Source source = {.part = {{x, (size_t)a.bytes}, {y, (size_t)b.bytes}}}; + return chunk_bytes(tree, &source, a.bytes + b.bytes, out); + } + Entry kids[FAN_MAX]; + List all = {0}; + size_t count = children(tree, a, height, kids); + bool ok = count != 0; + for (size_t i = 0; ok && i < count; ++i) ok = push(&all, kids[i]); + count = ok ? children(tree, b, height, kids) : 0; + ok = count != 0; + for (size_t i = 0; ok && i < count; ++i) ok = push(&all, kids[i]); + /* A small child carried in from a one-child parent is repaired here too. */ + ok = ok && repair(tree, &all, height - 1) && chunk_entries(tree, all.items, all.count, height, out); + free(all.items); + return ok; +} + +/* Replace [offset, offset + remove) of entry with insert. Appends the + * replacement nodes, all at entry's height, to out (possibly none). */ +static bool edit(Tree *tree, Entry entry, unsigned height, uint64_t offset, uint64_t remove, + PureSpan insert, List *out) { + if (offset > entry.bytes || remove > entry.bytes - offset) return false; + if (!height) { + const unsigned char *bytes = leaf_bytes(tree, entry); + if (!bytes) return false; + Source source = {.part = {{bytes, (size_t)offset}, insert, + {bytes + offset + remove, (size_t)(entry.bytes - offset - remove)}}}; + return chunk_bytes(tree, &source, entry.bytes - remove + insert.size, out); + } + Entry kids[FAN_MAX]; + size_t count = children(tree, entry, height, kids), first = count, final = count; + if (!count) return false; + uint64_t last = remove ? offset + remove - 1 : offset, start = 0, first_start = 0, final_start = 0; + for (size_t i = 0; i < count; ++i) { + uint64_t end = start + kids[i].bytes; + bool tail = i + 1 == count; + if (first == count && (offset < end || tail)) { first = i; first_start = start; } + if (first != count && (last < end || tail)) { final = i; final_start = start; break; } + start = end; + } + List middle = {0}, all = {0}; + bool ok = first == final ? + edit(tree, kids[first], height - 1, offset - first_start, remove, insert, &middle) : + edit(tree, kids[first], height - 1, offset - first_start, + first_start + kids[first].bytes - offset, insert, &middle) && + edit(tree, kids[final], height - 1, 0, offset + remove - final_start, (PureSpan){0}, &middle); + for (size_t i = 0; ok && i < first; ++i) ok = push(&all, kids[i]); + for (size_t i = 0; ok && i < middle.count; ++i) ok = push(&all, middle.items[i]); + for (size_t i = final + 1; ok && i < count; ++i) ok = push(&all, kids[i]); + free(middle.items); + ok = ok && repair(tree, &all, height - 1) && chunk_entries(tree, all.items, all.count, height, out); + free(all.items); + return ok; +} + +/* The new root of root with [offset, offset + remove) replaced by insert. */ +static bool rebuild(Tree *tree, Entry root, uint64_t offset, uint64_t remove, + PureSpan insert, Entry *out) { + unsigned height; + List list = {0}; + bool ok = root_height(tree, root, &height); + if (ok && !root.bytes) { + Source source = {.part = {insert}}; + ok = !offset && !remove && chunk_bytes(tree, &source, insert.size, &list); + } else if (ok) ok = edit(tree, root, height, offset, remove, insert, &list); + while (ok && list.count > 1) { + List up = {0}; + ok = height < HEIGHT_MAX && chunk_entries(tree, list.items, list.count, ++height, &up); + free(list.items); + list = up; + } + Entry result = ok && list.count ? list.items[0] : (Entry){0}; + free(list.items); + while (ok && height && result.bytes) { + Entry kids[FAN_MAX]; + size_t count = children(tree, result, height, kids); + if (count != 1) { ok = count > 1; break; } + result = kids[0]; + --height; + } + *out = result; + return ok; +} + +static bool flush(Writer *writer) { + if (writer->used && !writer->failed) { + digest_add(&writer->digest, writer->buffer, writer->used); + if (!write_at(writer->fd, writer->buffer, writer->used, writer->at - writer->used)) writer->failed = true; + } + writer->used = 0; + return !writer->failed; +} + +/* Zeros from..to; a failure only costs speed. */ +static void zero_fill(int fd, uint64_t from, uint64_t to) { + static const unsigned char zeros[BUFFER]; + while (from < to) { + size_t n = to - from < BUFFER ? (size_t)(to - from) : BUFFER; + if (!write_at(fd, zeros, n, from)) return; + from += n; + } +} + +/* Where the written zeros after a published end should reach. */ +static uint64_t tail_end(uint64_t end) { + return end + ((end - DATA) / 8 > TAIL_STEP ? (end - DATA) / 8 : TAIL_STEP); +} + +static void put_bytes(Writer *writer, const unsigned char *bytes, size_t length) { + while (length && !writer->failed) { + if (writer->used == BUFFER && !flush(writer)) return; + size_t n = BUFFER - writer->used < length ? BUFFER - writer->used : length; + memcpy(writer->buffer + writer->used, bytes, n); + writer->used += n; + writer->at += n; + bytes += n; + length -= n; + } +} + +static uint64_t put_branch(Writer *writer, const Entry *kids, size_t count, unsigned height) { + static const unsigned char zero[8]; + put_bytes(writer, zero, (size_t)((8 - writer->at % 8) % 8)); + uint64_t at = writer->at; + unsigned char header[8]; + memcpy(header, branch_magic, sizeof(branch_magic)); + header[4] = (unsigned char)count; + header[5] = (unsigned char)(count >> 8); + header[6] = (unsigned char)height; + header[7] = (unsigned char)(height >> 8); + put_bytes(writer, header, sizeof(header)); + for (size_t i = 0; i < count; ++i) { + unsigned char pair[16]; + encode64(pair, kids[i].ref); + encode64(pair + 8, kids[i].bytes); + put_bytes(writer, pair, sizeof(pair)); + } + return at; +} + +/* Append a subtree's arena nodes, children first, and set entry to the + * stored reference. The arena is not changed, so a failed commit leaves a + * draft intact. With relocate, stored nodes are copied too (compaction, or a + * draft whose base file was replaced by compaction). */ +static bool write_subtree(const Tree *tree, Writer *writer, Entry *entry, unsigned height, + bool relocate) { + if (!entry->bytes || (!(entry->ref & MEMORY_BIT) && !relocate)) return true; + if (!height) { + const unsigned char *bytes = leaf_bytes(tree, *entry); + if (!bytes) return false; + entry->ref = LEAF_BIT | writer->at; + put_bytes(writer, bytes, (size_t)entry->bytes); + return !writer->failed; + } + Entry kids[FAN_MAX]; + size_t count = children(tree, *entry, height, kids); + if (!count) return false; + for (size_t i = 0; i < count; ++i) + if (!write_subtree(tree, writer, &kids[i], height - 1, relocate)) return false; + entry->ref = put_branch(writer, kids, count, height); + return !writer->failed; +} + +static bool write_root(const Tree *tree, Writer *writer, Entry *root, bool relocate) { + unsigned height; + return root_height(tree, *root, &height) && write_subtree(tree, writer, root, height, relocate); +} + +static bool read_record(const Tree *tree, Entry catalog, uint64_t index, + unsigned char record[RECORD], Entry *document, int64_t *revision) { + unsigned height; + if (!root_height(tree, catalog, &height) || + !copy_range(tree, catalog, height, index * RECORD, RECORD, record)) return false; + *document = (Entry){decode64(record + KEY), decode64(record + KEY + 8)}; + uint64_t stored = decode64(record + KEY + 16); + *revision = (int64_t)stored; + return stored && stored <= (uint64_t)INT64_MAX && !document->ref == !document->bytes && + document->bytes <= PURE_OBJECT_LIMIT; +} + +/* Binary search over records ordered by their zero-padded key bytes. */ +static bool find(const Tree *tree, Entry catalog, const char *key, uint64_t *index, + bool *found, Entry *document, int64_t *revision) { + unsigned char wanted[KEY] = {0}, record[RECORD]; + memcpy(wanted, key, strlen(key)); + uint64_t low = 0, high = catalog.bytes / RECORD; + *found = false; + *document = (Entry){0}; + *revision = 0; + while (low < high) { + uint64_t middle = low + (high - low) / 2; + if (!read_record(tree, catalog, middle, record, document, revision)) return false; + int order = memcmp(wanted, record, KEY); + if (!order) { *index = middle; *found = true; return true; } + if (order < 0) high = middle; else low = middle + 1; + } + *document = (Entry){0}; + *revision = 0; + *index = low; + return true; +} + +static int create_temporary(int directory, char name[96]) { + for (unsigned attempt = 0; attempt < 64; ++attempt) { + if (temporary_sequence == UINT64_MAX) break; + snprintf(name, 96, ".pending-%ld-%" PRIu64, (long)getpid(), ++temporary_sequence); + int fd = openat(directory, name, O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600); + if (fd >= 0) return fd; + if (errno != EEXIST) break; + } + name[0] = 0; /* Never remove an existing file on a name collision. */ + return -1; +} + +/* Publish a fresh file with both slots pointing at head. It is synced whole + * before it is renamed into place, so its head names no commit bytes. */ +static bool install(int directory, char temporary[96], int fd, Head *head) { + unsigned char slot[SLOT]; + struct stat file; + head->start = head->end; + head->check = digest_end(&(Digest){0}); + encode_head(slot, head_magic, head); + zero_fill(fd, head->end, tail_end(head->end)); + bool ok = write_at(fd, slot, SLOT, 0) && write_at(fd, slot, SLOT, BLOCK) && sync_fd(fd) && !fstat(fd, &file); + if (close(fd)) ok = false; + ok = ok && !renameat(directory, temporary, directory, tree_name); + if (ok) temporary[0] = 0; + ok = ok && sync_fd(directory); + if (ok) remember(&file, head, DURABLE); + return ok; +} + +/* Rewrite only reachable nodes. Open readers keep their mapping of the + * previous file. A failure leaves the published file untouched. */ +static void compact(int directory, int source, const struct stat *file, const Head *head) { + Tree tree; + if (!map_tree(source, file, head, &tree)) return; + char temporary[96]; + int fd = create_temporary(directory, temporary); + Writer *writer = fd >= 0 ? malloc(sizeof(*writer)) : NULL; + uint64_t count = head->catalog_bytes / RECORD; + unsigned char *records = count ? malloc((size_t)(count * RECORD)) : NULL; + Entry catalog = {head->catalog, head->catalog_bytes}; + unsigned height; + bool ok = writer && (!count || records) && root_height(&tree, catalog, &height) && + (!count || copy_range(&tree, catalog, height, 0, count * RECORD, records)); + if (writer) *writer = (Writer){.fd = fd, .at = DATA}; + for (uint64_t i = 0; ok && i < count; ++i) { + unsigned char *record = records + i * RECORD; + Entry document = {decode64(record + KEY), decode64(record + KEY + 8)}; + unsigned document_height; + ok = root_height(&tree, document, &document_height) && + write_subtree(&tree, writer, &document, document_height, true); + encode64(record + KEY, document.ref); + } + Entry fresh = {0}; + ok = ok && rebuild(&tree, (Entry){0}, 0, 0, (PureSpan){records, (size_t)(count * RECORD)}, &fresh) && + write_root(&tree, writer, &fresh, false) && flush(writer); + if (ok) { + Head compacted = {head->seq + 1, writer->at, fresh.ref, fresh.bytes, writer->at - DATA, 0, 0}; + ok = install(directory, temporary, fd, &compacted); + fd = -1; + } + if (fd >= 0) close(fd); + if (*temporary) unlinkat(directory, temporary, 0); + free(records); + free(writer); + release_tree(&tree); +} + +static bool lock_writer(int lock, bool wait) { + struct flock hold = {.l_type = F_WRLCK, .l_whence = SEEK_SET, .l_start = 0, .l_len = 0}; + int rc; + do { rc = fcntl(lock, wait ? F_SETLKW : F_SETLK, &hold); } while (rc && wait && errno == EINTR); + return rc == 0; +} + +PureResult pure_store_init(const char *path) { + if (!path || !*path) return PURE_STORE_ERROR; + bool created = mkdir(path, 0700) == 0; + if (!created && errno != EEXIST) return PURE_STORE_ERROR; + int directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (directory < 0) return PURE_STORE_ERROR; /* Never reinterpret an old DB file. */ + pthread_mutex_lock(&writers); + int lock = openat(directory, ".writer.lock", O_RDWR | O_CREAT | O_NOFOLLOW | O_CLOEXEC, 0600); + struct stat info; + bool ok = lock >= 0 && fstat(lock, &info) == 0 && S_ISREG(info.st_mode) && lock_writer(lock, true); + if (ok) { + Head head; + struct stat file; + int fd = openat(directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd >= 0) { + ok = read_head(fd, &file, true, &head, NULL); /* An unreadable file is never replaced. */ + close(fd); + } else if (errno != ENOENT) ok = false; + else { + char temporary[96]; + int created_fd = create_temporary(directory, temporary); + head = (Head){.seq = 1, .end = DATA}; + ok = created_fd >= 0 && install(directory, temporary, created_fd, &head); + if (*temporary) unlinkat(directory, temporary, 0); + } + } + ok = ok && sync_fd(lock) && sync_fd(directory); + if (lock >= 0) close(lock); + pthread_mutex_unlock(&writers); + close(directory); + if (ok && created) { + char *parent = strdup(path); + if (!parent) return PURE_STORE_ERROR; + size_t length = strlen(parent); + while (length > 1 && parent[length - 1] == '/') parent[--length] = 0; + char *slash = strrchr(parent, '/'); + if (!slash) strcpy(parent, "."); + else if (slash == parent) parent[1] = 0; + else *slash = 0; + int fd = open(parent, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + free(parent); + ok = fd >= 0 && sync_fd(fd); + if (fd >= 0) close(fd); + } + return ok ? PURE_OK : PURE_STORE_ERROR; +} + +void pure_store_release(PureStoredView *view) { + if (view->mapped_size) munmap(view->mapping, view->mapped_size); + *view = (PureStoredView){0}; +} + +PureResult pure_store_select(const char *path, const char *key, size_t offset, + size_t length, bool whole, size_t minimum, + size_t available, PureStoredView *out) { + *out = (PureStoredView){0}; + if (!valid_key(key)) return PURE_STORE_ERROR; + int directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (directory < 0) return PURE_STORE_ERROR; + int fd = openat(directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + close(directory); + if (fd < 0) return PURE_STORE_ERROR; + Tree tree = {0}; + Head head; + struct stat file; + Entry document; + uint64_t index; + int64_t revision; + bool found; + unsigned height; + PureResult result = PURE_STORE_ERROR; + if (!read_head(fd, &file, false, &head, NULL) || !map_tree(fd, &file, &head, &tree) || + !find(&tree, (Entry){head.catalog, head.catalog_bytes}, key, &index, &found, + &document, &revision)) goto finish; + if (!found) { result = PURE_NOT_FOUND; goto finish; } + size_t size = (size_t)document.bytes; + if (size < minimum || offset > size) { result = PURE_LIMIT; goto finish; } + if (whole) length = size - offset; + if (length > size - offset) { result = PURE_LIMIT; goto finish; } + if (length > available) { result = PURE_BUDGET; goto finish; } + if (length) { + if (!root_height(&tree, document, &height)) goto finish; + const unsigned char *direct = single_leaf(&tree, document, height, offset, length); + void *mapping; + size_t mapped_size, prefix = 0; + if (direct) { + /* One extent: map it in place, as the flat store does. */ + long page = sysconf(_SC_PAGESIZE); + if (page <= 0) goto finish; + size_t start = (size_t)(direct - tree.map), aligned = start - start % (size_t)page; + prefix = start - aligned; + mapped_size = prefix + length; + mapping = mmap(NULL, mapped_size, PROT_READ, MAP_PRIVATE, fd, (off_t)aligned); + if (mapping == MAP_FAILED) goto finish; + } else { + /* A contiguous view of several extents is assembled for this API. */ + mapped_size = length; + mapping = mmap(NULL, mapped_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) goto finish; + if (!copy_range(&tree, document, height, offset, length, mapping) || + mprotect(mapping, mapped_size, PROT_READ)) { + munmap(mapping, mapped_size); + goto finish; + } + } + out->mapping = mapping; + out->mapped_size = mapped_size; + out->body = (PureSpan){(const unsigned char *)mapping + prefix, length}; + } + out->revision = revision; + out->object_size = size; + result = PURE_OK; +finish: + release_tree(&tree); + close(fd); + return result; +} + +PureResult pure_store_get(const char *path, const char *key, size_t available, PureStoredView *out) { + return pure_store_select(path, key, 0, 0, true, 0, available, out); +} + +PureResult pure_store_get_range(const char *path, const char *key, size_t offset, + size_t length, size_t available, PureStoredView *out) { + return pure_store_select(path, key, offset, length, false, 0, available, out); +} + +/* A locked writer over the current published file. */ +typedef struct { + int directory, lock, fd; + Head head; + uint64_t slots[2]; /* the sequences of the slots' heads */ + Tree tree; + struct stat file; + bool turn; /* holds this process's writer turn */ +} Session; + +/* Keep written zeros after the published end, where commits and their + * unpublished leftovers go. A sync of blocks the file already has writes + * only them; a new size or new blocks add a journal commit (on an SD card + * 2.4 ms against 6 ms per commit). The zeros become durable with the next + * commit's sync. */ +static void prepare_tail(Session *session) { + uint64_t size = (uint64_t)session->file.st_size, end = session->head.end; + if (size < end + TAIL_MIN) zero_fill(session->fd, size > end ? size : end, tail_end(end)); +} + +static void close_session(Session *session) { + release_tree(&session->tree); + if (session->fd >= 0) close(session->fd); + if (session->lock >= 0) close(session->lock); + if (session->directory >= 0) close(session->directory); + if (session->turn) pthread_mutex_unlock(&writers); + session->turn = false; +} + +static PureResult open_session(const char *path, Session *session) { + *session = (Session){.directory = -1, .lock = -1, .fd = -1}; + pthread_mutex_lock(&writers); /* another thread of this process waits, not fails */ + session->turn = true; + session->directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (session->directory < 0) return PURE_STORE_ERROR; + session->lock = openat(session->directory, ".writer.lock", O_RDWR | O_NOFOLLOW | O_CLOEXEC); + struct stat info; + if (session->lock < 0 || fstat(session->lock, &info) || !S_ISREG(info.st_mode)) return PURE_STORE_ERROR; + if (!lock_writer(session->lock, false)) + return errno == EACCES || errno == EAGAIN ? PURE_BUSY : PURE_STORE_ERROR; + session->fd = openat(session->directory, tree_name, O_RDWR | O_NOFOLLOW | O_CLOEXEC); + if (session->fd < 0 || !read_head(session->fd, &session->file, true, &session->head, session->slots)) + return PURE_STORE_ERROR; + /* A head this process did not make durable may be the last write of a + * writer that died before its sync: make it durable before building on it. */ + if (recall(&session->file, &session->head) != DURABLE) { + if (!sync_data(session->fd)) return PURE_STORE_ERROR; + remember(&session->file, &session->head, DURABLE); + } + prepare_tail(session); + return map_tree(session->fd, &session->file, &session->head, &session->tree) ? PURE_OK : PURE_STORE_ERROR; +} + +/* A commit's bytes start after room for its record at the published end. */ +static uint64_t commit_start(const Session *session) { return session->head.end + SLOT; } + +/* The documents' new nodes are already written at writer. Record each in + * the catalog (revisions come in as the base and leave as the new one), then + * write the head as the record before them and sync once: every document + * becomes visible at once, or none does. Readers of this process see the + * head only once it is durable. */ +#ifdef PURE_STORE_TEST +static unsigned long publications; +unsigned long pure_tree_test_publications(void) { return publications; } +#endif + +static PureResult publish(Session *session, Writer *writer, size_t count, const char *const *keys, + const Entry *documents, int64_t *revisions) { +#ifdef PURE_STORE_TEST + ++publications; +#endif + Entry catalog = {session->head.catalog, session->head.catalog_bytes}; + bool written = true; + for (size_t i = 0; written && i < count; ++i) { + unsigned char record[RECORD] = {0}; + uint64_t index; + bool found; + Entry current, updated; + int64_t stored; + memcpy(record, keys[i], strlen(keys[i])); + encode64(record + KEY, documents[i].ref); + encode64(record + KEY + 8, documents[i].bytes); + encode64(record + KEY + 16, (uint64_t)revisions[i] + 1); + written = find(&session->tree, catalog, keys[i], &index, &found, ¤t, &stored) && + rebuild(&session->tree, catalog, index * RECORD, found ? RECORD : 0, + (PureSpan){record, RECORD}, &updated); + catalog = updated; + } + written = written && write_root(&session->tree, writer, &catalog, false) && flush(writer); + /* Unpublished bytes stay in the tail; the next commit overwrites them. */ + if (!written || fault(PURE_AFTER_HEADER) || fault(PURE_BEFORE_FILE_SYNC) || fault(PURE_BEFORE_RENAME)) + return PURE_STORE_ERROR; + Head published = {session->head.seq + 1, writer->at, catalog.ref, catalog.bytes, session->head.live, + commit_start(session), digest_end(&writer->digest)}; + unsigned char record[SLOT], slot[SLOT]; + encode_head(record, record_magic, &published); + encode_head(slot, head_magic, &published); + /* Every few commits the head goes into the older slot as well: one more + * range for that sync, and fewer records for a reader to follow. */ + uint64_t newer = session->slots[0] > session->slots[1] ? session->slots[0] : session->slots[1]; + unsigned older = session->slots[0] > session->slots[1]; + bool checkpoint = published.seq - newer >= CHECKPOINT; + for (size_t i = 0; i < count; ++i) ++revisions[i]; /* Past this point the head may already be visible. */ + remember(&session->file, &published, PUBLISHING); + bool durable = write_at(session->fd, record, SLOT, published.start - SLOT) && + (!checkpoint || write_at(session->fd, slot, SLOT, older * BLOCK)) && + !fault(PURE_BEFORE_DIR_SYNC) && sync_data(session->fd); + remember(&session->file, &published, durable ? DURABLE : CHECKED); + if (!durable) return PURE_UNCERTAIN; + if (published.end - DATA > 4 * published.live + COMPACT_SLACK && !fstat(session->fd, &session->file)) + compact(session->directory, session->fd, &session->file, &published); + return PURE_OK; +} + +static PureResult mutate(const char *path, const char *key, int64_t expected, + PureSpan body, bool splice, size_t offset, size_t remove, + size_t max_size, int64_t *revision) { + *revision = 0; + if (!valid_key(key) || (body.size && !body.data)) return PURE_STORE_ERROR; + if (body.size > PURE_OBJECT_LIMIT || max_size > PURE_OBJECT_LIMIT || expected < 0) return PURE_LIMIT; + Session session; + Writer *writer = NULL; + Entry document, next; + uint64_t index; + bool found; + PureResult result = open_session(path, &session); + if (result != PURE_OK) goto finish; + result = PURE_STORE_ERROR; + if (!find(&session.tree, (Entry){session.head.catalog, session.head.catalog_bytes}, key, + &index, &found, &document, revision)) goto finish; + if (*revision != expected) { result = PURE_CONFLICT; goto finish; } + if (*revision == INT64_MAX) { result = PURE_LIMIT; goto finish; } + uint64_t old_size = document.bytes; + if (splice && (offset > old_size || remove > old_size - offset || + old_size - remove > max_size || + body.size > max_size - (old_size - remove))) { result = PURE_LIMIT; goto finish; } + writer = malloc(sizeof(*writer)); + if (!writer || !rebuild(&session.tree, splice ? document : (Entry){0}, splice ? offset : 0, + splice ? remove : 0, body, &next)) goto finish; + *writer = (Writer){.fd = session.fd, .at = commit_start(&session)}; + if (!write_root(&session.tree, writer, &next, false)) goto finish; + result = publish(&session, writer, 1, &key, &next, revision); +finish: + free(writer); + close_session(&session); + return result; +} + +/* Keys come from the catalog in ascending byte order. Runtime values are not + * documents and are left out. */ +PureResult pure_store_keys(const char *path, bool (*visit)(void *, const char *key), + void *context) { + int directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (directory < 0) return PURE_STORE_ERROR; + int fd = openat(directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + close(directory); + if (fd < 0) return PURE_STORE_ERROR; + Tree tree = {0}; + Head head; + struct stat file; + PureResult result = PURE_STORE_ERROR; + if (read_head(fd, &file, false, &head, NULL) && map_tree(fd, &file, &head, &tree)) { + Entry catalog = {head.catalog, head.catalog_bytes}, document; + unsigned char record[RECORD]; + char key[KEY + 1] = {0}; + int64_t revision; + result = PURE_OK; + for (uint64_t i = 0; i < catalog.bytes / RECORD; ++i) { + if (!read_record(&tree, catalog, i, record, &document, &revision)) { result = PURE_STORE_ERROR; break; } + memcpy(key, record, KEY); + if (runtime_key(key)) continue; + if (!valid_key(key)) { result = PURE_STORE_ERROR; break; } + if (!visit(context, key)) break; + } + } + release_tree(&tree); + close(fd); + return result; +} + +/* A store kept open by one owner thread: its current file and a mapping of + * its newest version, shared by the drafts opened through it. */ +struct PureStore { + char *path; + int directory, fd; + dev_t device; + ino_t inode; + Head head; + Mapping *mapping; + bool pinned; /* drafts open at the version of the last check */ +}; + +struct PureDraft { + PureStore *store; /* the handle it was opened through, or NULL */ + char *path; /* owned unless opened through a store handle */ + char key[KEY + 1]; + Tree tree; /* the base file's mapping plus this draft's node arena */ + dev_t device; /* identity of the base file; compaction replaces it */ + ino_t inode; + Entry root; + int64_t revision; + size_t live; /* arena bytes reachable at the last collection */ + bool held; /* a transaction still reads earlier versions */ +}; + +/* Rebase the draft on the committed version of its key. */ +static PureResult load_draft(PureDraft *draft) { + int directory = open(draft->path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (directory < 0) return PURE_STORE_ERROR; + int fd = openat(directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + close(directory); + if (fd < 0) return PURE_STORE_ERROR; + Tree tree = {0}; + Head head; + struct stat info; + Entry document; + uint64_t index; + int64_t revision; + bool found; + bool ok = read_head(fd, &info, false, &head, NULL) && map_tree(fd, &info, &head, &tree) && + find(&tree, (Entry){head.catalog, head.catalog_bytes}, draft->key, &index, &found, + &document, &revision); + close(fd); + if (!ok) { release_tree(&tree); return PURE_STORE_ERROR; } + release_tree(&draft->tree); + draft->tree = tree; + draft->device = info.st_dev; + draft->inode = info.st_ino; + draft->root = document; + draft->revision = revision; + draft->live = 0; + return PURE_OK; +} + +PureResult pure_draft_open(const char *path, const char *key, PureDraft **out) { + *out = NULL; + if (!path || !(valid_key(key) || runtime_key(key))) return PURE_INVALID; + PureDraft *draft = calloc(1, sizeof(*draft)); + if (!draft || !(draft->path = strdup(path))) { free(draft); return PURE_ALLOCATION; } + memcpy(draft->key, key, strlen(key) + 1); + PureResult result = load_draft(draft); + if (result != PURE_OK) { pure_draft_close(draft); return result; } + *out = draft; + return PURE_OK; +} + +PureResult pure_state_open(const char *path, const char *name, PureDraft **out) { + char key[KEY + 2]; + *out = NULL; + if (!valid_key(name) || strlen(name) >= KEY) return PURE_INVALID; + snprintf(key, sizeof(key), ".%s", name); + return pure_draft_open(path, key, out); +} + +PureDraft *pure_draft_new(void) { + return calloc(1, sizeof(PureDraft)); /* no path: nothing mapped, nothing to commit */ +} + +PureDraft *pure_draft_freeze(const PureDraft *draft) { + if (!draft->tree.shared || (draft->root.ref & MEMORY_BIT)) return NULL; + PureDraft *frozen = calloc(1, sizeof(*frozen)); + if (!frozen) return NULL; + Mapping *mapping = draft->tree.shared; + __atomic_add_fetch(&mapping->refs, 1, __ATOMIC_RELAXED); + *frozen = (PureDraft){.tree = {.map = mapping->map, .end = mapping->end, .shared = mapping}, + .device = draft->device, .inode = draft->inode, .root = draft->root, + .revision = draft->revision}; + memcpy(frozen->key, draft->key, sizeof(frozen->key)); + return frozen; +} + +void pure_draft_close(PureDraft *draft) { + if (!draft) return; + release_tree(&draft->tree); + if (!draft->store) free(draft->path); + free(draft); +} + +/* Bring the handle to the newest published version: an unchanged store costs + * one system call (the mapping shows the head slots); a new commit maps + * again, a compaction reopens. */ +static PureResult refresh(PureStore *store) { + struct stat info; + if (fstatat(store->directory, tree_name, &info, AT_SYMLINK_NOFOLLOW) || !S_ISREG(info.st_mode)) + return PURE_STORE_ERROR; + if (store->fd < 0 || info.st_dev != store->device || info.st_ino != store->inode) { + int fd = openat(store->directory, tree_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd < 0 || fstat(fd, &info)) { if (fd >= 0) close(fd); return PURE_STORE_ERROR; } + if (store->fd >= 0) close(store->fd); + store->fd = fd; + store->device = info.st_dev; + store->inode = info.st_ino; + store->head = (Head){0}; + } + if (store->head.seq) { + /* The next commit's record follows the head's bytes; a slot shows a + * newer head only for a commit of the earlier format. */ + const unsigned char *map = store->mapping->map; + unsigned char next[16]; + bool newer = decode64(map + 8) > store->head.seq || decode64(map + BLOCK + 8) > store->head.seq || + (read_at(store->fd, next, sizeof(next), store->head.end) && !memcmp(next, record_magic, 8) && + decode64(next + 8) == store->head.seq + 1); + if (!newer) return PURE_OK; + } + Head head; + if (!read_head(store->fd, &info, false, &head, NULL)) return PURE_STORE_ERROR; + if (store->mapping && head.seq == store->head.seq && head.check == store->head.check) return PURE_OK; + Tree tree; + Mapping *mapping = malloc(sizeof(*mapping)); + if (!mapping) return PURE_ALLOCATION; + if (!map_tree(store->fd, &info, &head, &tree)) { free(mapping); return PURE_STORE_ERROR; } + *mapping = (Mapping){tree.map, tree.end, 1}; + release_mapping(store->mapping); + store->mapping = mapping; + store->head = head; + return PURE_OK; +} + +/* Base a draft on the handle's newest version of its key, sharing the mapping. + * On failure the draft is unchanged. */ +static PureResult rebase(PureDraft *draft) { + PureStore *store = draft->store; + Mapping *mapping = store->mapping; + Tree tree = {.map = mapping->map, .end = mapping->end, .shared = mapping}; + Entry document; + uint64_t index; + int64_t revision; + bool found; + if (!find(&tree, (Entry){store->head.catalog, store->head.catalog_bytes}, draft->key, &index, &found, &document, + &revision)) return PURE_STORE_ERROR; + __atomic_add_fetch(&mapping->refs, 1, __ATOMIC_RELAXED); + release_tree(&draft->tree); + draft->tree = tree; + draft->device = store->device; + draft->inode = store->inode; + draft->root = document; + draft->revision = revision; + draft->live = 0; + return PURE_OK; +} + +PureResult pure_store_open(const char *path, PureStore **out) { + *out = NULL; + PureStore *store = calloc(1, sizeof(*store)); + if (!store || !path || !(store->path = strdup(path))) { free(store); return PURE_ALLOCATION; } + store->fd = -1; + store->directory = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + PureResult result = store->directory < 0 ? PURE_STORE_ERROR : refresh(store); + if (result != PURE_OK) { pure_store_close(store); return result; } + *out = store; + return PURE_OK; +} + +void pure_store_close(PureStore *store) { + if (!store) return; + release_mapping(store->mapping); + if (store->fd >= 0) close(store->fd); + if (store->directory >= 0) close(store->directory); + free(store->path); + free(store); +} + +PureMapping *pure_draft_mapping(const PureDraft *draft) { + Mapping *mapping = draft->tree.shared; + if (mapping) __atomic_add_fetch(&mapping->refs, 1, __ATOMIC_RELAXED); + return mapping; +} + +bool pure_mapping_holds(const PureMapping *mapping, PureSpan span) { + uintptr_t base = (uintptr_t)mapping->map, at = (uintptr_t)span.data; + return !span.size || (span.data && at >= base && at - base <= mapping->end && span.size <= mapping->end - (at - base)); +} + +void pure_mapping_release(PureMapping *mapping) { release_mapping(mapping); } + +PureResult pure_store_pin(PureStore *store, bool pinned) { + store->pinned = false; + PureResult result = pinned ? refresh(store) : PURE_OK; + store->pinned = pinned && result == PURE_OK; + return result; +} + +static PureResult open_through(PureStore *store, const char *key, PureDraft **out) { + *out = NULL; + PureResult result = store->pinned ? PURE_OK : refresh(store); + if (result != PURE_OK) return result; + PureDraft *draft = calloc(1, sizeof(*draft)); + if (!draft) return PURE_ALLOCATION; + *draft = (PureDraft){.store = store, .path = store->path}; + memcpy(draft->key, key, strlen(key) + 1); + result = rebase(draft); + if (result != PURE_OK) { free(draft); return result; } + *out = draft; + return PURE_OK; +} + +PureResult pure_store_draft(PureStore *store, const char *key, PureDraft **out) { + *out = NULL; + return store && valid_key(key) ? open_through(store, key, out) : PURE_INVALID; +} + +PureResult pure_store_state(PureStore *store, const char *name, PureDraft **out) { + char key[KEY + 2]; + *out = NULL; + if (!store || !valid_key(name) || strlen(name) >= KEY) return PURE_INVALID; + snprintf(key, sizeof(key), ".%s", name); + return open_through(store, key, out); +} + +size_t pure_draft_size(const PureDraft *draft) { return (size_t)draft->root.bytes; } + +int64_t pure_draft_revision(const PureDraft *draft) { return draft->revision; } + +static size_t mark(const Tree *tree, Entry entry, unsigned char *marks) { + if (!(entry.ref & MEMORY_BIT)) return 0; + size_t at = (size_t)(entry.ref & LOCATION); + if (marks[at]) return 0; + marks[at] = 1; + const Node *node = &tree->nodes[at]; + size_t bytes = node_bytes(node); + if (!node->bytes) + for (size_t i = 0; i < node->size; ++i) bytes += mark(tree, node->children[i], marks); + return bytes; +} + +/* Every edit copies a leaf and path, so reclaim arena nodes the root no + * longer reaches once they outweigh the reachable ones (and 32 KiB). */ +static void collect(PureDraft *draft) { + Tree *tree = &draft->tree; + if (draft->held) return; + size_t allowance = draft->live > 32768 ? draft->live : 32768; + if (tree->bytes <= draft->live || tree->bytes - draft->live <= allowance) return; + unsigned char *marks = calloc(tree->count, 1); + if (!marks) return; + draft->live = mark(tree, draft->root, marks); + for (size_t i = 0; i < tree->count; ++i) { + Node *node = &tree->nodes[i]; + if (marks[i] || node->height == FREE) continue; + tree->bytes -= node_bytes(node); + free(node->bytes); + free(node->children); + *node = (Node){.size = tree->free_head, .height = FREE}; + tree->free_head = i + 1; + ++tree->free_count; + } + free(marks); +} + +PureResult pure_draft_splice(PureDraft *draft, size_t offset, size_t remove, PureSpan insert) { + uint64_t size = draft->root.bytes; + if (insert.size && !insert.data) return PURE_INVALID; + if (offset > size || remove > size - offset || insert.size > PURE_OBJECT_LIMIT - (size - remove)) + return PURE_LIMIT; + if (!remove && !insert.size) return PURE_OK; + Entry next; + bool ok = rebuild(&draft->tree, draft->root, offset, remove, insert, &next); + if (ok) draft->root = next; + collect(draft); + return ok ? PURE_OK : PURE_STORE_ERROR; +} + +PureVersion pure_draft_version(const PureDraft *draft) { + return (PureVersion){draft->root.ref, draft->root.bytes}; +} + +PureResult pure_draft_read_version(const PureDraft *draft, PureVersion version, size_t offset, + size_t length, bool (*visit)(void *, PureSpan), void *context) { + Entry root = {version.ref, version.bytes}; + unsigned height; + if (offset > root.bytes || length > root.bytes - offset) return PURE_LIMIT; + if (!length) return PURE_OK; + if (!root_height(&draft->tree, root, &height)) return PURE_STORE_ERROR; + return visit_range(&draft->tree, root, height, offset, length, visit, context) ? + PURE_OK : PURE_STORE_ERROR; +} + +PureResult pure_draft_read(const PureDraft *draft, size_t offset, size_t length, + bool (*visit)(void *, PureSpan), void *context) { + return pure_draft_read_version(draft, pure_draft_version(draft), offset, length, visit, context); +} + +void pure_draft_restore(PureDraft *draft, PureVersion version) { + draft->root = (Entry){version.ref, version.bytes}; +} + +void pure_draft_hold(PureDraft *draft, bool held) { + draft->held = held; + if (!held) collect(draft); +} + +/* ---- Group commits ------------------------------------------------------ + * Threads that commit at the same time share one commit: they queue, the + * first becomes the leader and publishes every queued request of the same + * store with one head (one pair of syncs); the others wait for its result. + * Each request stays all or nothing on its own: a request with a stale draft + * is left out, and so is one whose key an earlier request of the same group + * already changes. The leader touches the waiters' drafts and handles only + * while they wait. */ +typedef struct Waiter { + PureDraft *const *drafts; + size_t count; + int64_t *revisions; + PureResult result; + bool accepted, done; + struct Waiter *next; +} Waiter; + +static struct { + pthread_mutex_t lock; + pthread_cond_t finished, joined; + Waiter *head, *tail; + size_t queued, expected; /* expected: how many requests groups have held lately */ + bool leading; + unsigned long groups, requests; +} group = {PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, PTHREAD_COND_INITIALIZER, NULL, NULL, 0, 1, false, 0, 0}; + +void pure_store_group_counts(unsigned long *groups, unsigned long *requests) { + pthread_mutex_lock(&group.lock); + *groups = group.groups; + *requests = group.requests; + pthread_mutex_unlock(&group.lock); +} + +enum { GATHER_NS = 2000000 }; + +/* Publish the queued requests of the store at path in one session. */ +static void commit_group(const char *path, Waiter *first) { + size_t total = 0; + for (Waiter *w = first; w; w = w->next) + if (!strcmp(w->drafts[0]->path, path)) { total += w->count; w->accepted = false; } + Session session; + Writer *writer = NULL; + Entry *roots = malloc(total * sizeof(*roots)); + const char **keys = malloc(total * sizeof(*keys)); + int64_t *revisions = malloc(total * sizeof(*revisions)); + PureDraft **drafts = malloc(total * sizeof(*drafts)); + bool allocated = roots && keys && revisions && drafts, checked = false; + PureResult result = allocated ? open_session(path, &session) : PURE_ALLOCATION; + size_t count = 0; + if (result == PURE_OK) { + /* A request holds when each of its keys is at its draft's base + * revision and no earlier request of this group changes it. */ + for (Waiter *w = first; w; w = w->next) { + if (strcmp(w->drafts[0]->path, path)) continue; + w->result = PURE_OK; + for (size_t i = 0; i < w->count && w->result == PURE_OK; ++i) { + PureDraft *d = w->drafts[i]; + Entry current; + uint64_t index; + int64_t stored; + bool found; + if (!find(&session.tree, (Entry){session.head.catalog, session.head.catalog_bytes}, d->key, &index, + &found, ¤t, &stored)) { w->result = PURE_STORE_ERROR; break; } + for (size_t k = 0; k < count; ++k) + if (!strcmp(keys[k], d->key)) { ++stored; break; } /* it moves with the earlier request */ + if (stored != d->revision) { w->revisions[i] = stored; w->result = PURE_CONFLICT; } + else if (stored == INT64_MAX) w->result = PURE_LIMIT; + } + if (w->result != PURE_OK) continue; + w->accepted = true; + for (size_t i = 0; i < w->count; ++i) { + drafts[count] = w->drafts[i]; + roots[count] = w->drafts[i]->root; + keys[count] = w->drafts[i]->key; + revisions[count++] = w->drafts[i]->revision; + } + } + checked = true; + if (count) { + result = PURE_ALLOCATION; + if ((writer = malloc(sizeof(*writer)))) { + *writer = (Writer){.fd = session.fd, .at = commit_start(&session)}; + bool written = true; + for (size_t i = 0; i < count && written; ++i) { + /* After compaction a draft's stored references name the old file. */ + bool relocate = session.file.st_dev != drafts[i]->device || session.file.st_ino != drafts[i]->inode; + written = write_root(&drafts[i]->tree, writer, &roots[i], relocate); + } + result = written ? publish(&session, writer, count, keys, roots, revisions) : PURE_STORE_ERROR; + } + } + /* Rebase while still holding the writer lock; the arenas are no longer + * needed. Drafts of a store handle share one new mapping. If remapping + * fails a draft keeps equal content at its new revision. */ + PureStore *refreshed[8] = {NULL}; + size_t fresh = 0; + for (size_t i = 0; result == PURE_OK && i < count; ++i) { + PureStore *store = drafts[i]->store; + bool rebased; + if (store) { + bool seen = false; + for (size_t k = 0; k < fresh && !seen; ++k) seen = refreshed[k] == store; + if (!seen && refresh(store) == PURE_OK) { + seen = true; + if (fresh < 8) refreshed[fresh++] = store; + } + rebased = seen && rebase(drafts[i]) == PURE_OK; + } else rebased = load_draft(drafts[i]) == PURE_OK; + if (!rebased) drafts[i]->revision = revisions[i]; + } + } + /* Before the checks every request shares the failure; after them an + * accepted request takes the publication's result and new revisions, and + * a refused one keeps its own result. */ + size_t at = 0; + for (Waiter *w = first; w; w = w->next) { + if (strcmp(w->drafts[0]->path, path)) continue; + if (!checked) { w->result = result; continue; } + if (!w->accepted) continue; + for (size_t i = 0; i < w->count; ++i) w->revisions[i] = revisions[at++]; + w->result = result; + } + free(writer); + free(roots); + free((void *)keys); + free(revisions); + free(drafts); + if (allocated) close_session(&session); +} + +PureResult pure_draft_commit_all(PureDraft *const *drafts, size_t count, int64_t *revisions) { + for (size_t i = 0; i < count; ++i) { + revisions[i] = drafts[i]->revision; + if (!drafts[i]->path || strcmp(drafts[i]->path, drafts[0]->path)) return PURE_INVALID; + for (size_t k = 0; k < i; ++k) if (!strcmp(drafts[k]->key, drafts[i]->key)) return PURE_INVALID; + } + if (!count) return PURE_OK; + Waiter me = {.drafts = drafts, .count = count, .revisions = revisions}; + pthread_mutex_lock(&group.lock); + if (group.tail) group.tail->next = &me; else group.head = &me; + group.tail = &me; + ++group.queued; + pthread_cond_signal(&group.joined); + while (!me.done) { + if (group.leading) { pthread_cond_wait(&group.finished, &group.lock); continue; } + group.leading = true; + /* When several threads have been committing together, give the others + * a moment (at most 2 ms) to join, or they would miss this group and + * wait for the next one: a sync costs far more. One thread never waits. */ + if (group.expected > 1 && group.queued < group.expected) { + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_nsec += GATHER_NS; + if (deadline.tv_nsec >= 1000000000) { ++deadline.tv_sec; deadline.tv_nsec -= 1000000000; } + while (group.queued < group.expected && + pthread_cond_timedwait(&group.joined, &group.lock, &deadline) == 0) {} + } + Waiter *batch = group.head; + size_t size = group.queued; + group.head = group.tail = NULL; + group.queued = 0; + ++group.groups; + group.requests += size; + /* Remember the largest recent group, forgetting one member per group. */ + group.expected = size > group.expected - 1 ? size : group.expected - 1; + if (!group.expected) group.expected = 1; + pthread_mutex_unlock(&group.lock); + /* One session per store among the queued requests. */ + for (Waiter *w = batch; w; w = w->next) { + bool earlier = false; + for (Waiter *v = batch; v != w && !earlier; v = v->next) earlier = !strcmp(v->drafts[0]->path, w->drafts[0]->path); + if (!earlier) commit_group(w->drafts[0]->path, batch); + } + pthread_mutex_lock(&group.lock); + for (Waiter *w = batch, *next; w; w = next) { next = w->next; w->done = true; } + group.leading = false; + pthread_cond_broadcast(&group.finished); + } + pthread_mutex_unlock(&group.lock); + return me.result; +} + +PureResult pure_draft_commit(PureDraft *draft, int64_t *revision) { + return pure_draft_commit_all(&draft, 1, revision); +} + +PureResult pure_store_put(const char *path, const char *key, int64_t expected, + PureSpan body, int64_t *revision) { + return mutate(path, key, expected, body, false, 0, 0, PURE_OBJECT_LIMIT, revision); +} + +PureResult pure_store_splice(const char *path, const char *key, int64_t expected, + size_t offset, size_t remove, PureSpan insert, + size_t max_size, int64_t *revision) { + return mutate(path, key, expected, insert, true, offset, remove, max_size, revision); +} + +#ifdef PURE_STORE_TEST +static bool shape(const Tree *tree, Entry entry, unsigned height, bool root, + uint64_t *leaves, uint64_t *small) { + if (!height) { + ++*leaves; + if (!root && entry.bytes < LEAF_MIN) ++*small; + return leaf_bytes(tree, entry) != NULL; + } + Entry kids[FAN_MAX]; + size_t count = children(tree, entry, height, kids); + if (!count || (root && count < 2)) return false; + if (!root && count < FAN_MIN) ++*small; + for (size_t i = 0; i < count; ++i) + if (!shape(tree, kids[i], height - 1, false, leaves, small)) return false; + return true; +} + +/* Test-only: live arena nodes and their payload bytes. */ +void pure_draft_test_arena(const PureDraft *draft, size_t *nodes, size_t *bytes) { + *nodes = *bytes = 0; + for (size_t i = 0; i < draft->tree.count; ++i) { + const Node *node = &draft->tree.nodes[i]; + if (node->height == FREE) continue; + ++*nodes; + *bytes += node->bytes ? node->size : node->size * sizeof(Entry); + } +} + +/* Test-only: the stored height, leaf count and number of underfull nodes. */ +bool pure_tree_test_shape(const char *path, const char *key, unsigned *height, + uint64_t *leaves, uint64_t *small, uint64_t *file_end) { + int directory = open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + int fd = directory >= 0 ? openat(directory, tree_name, O_RDONLY | O_CLOEXEC) : -1; + if (directory >= 0) close(directory); + Tree tree = {0}; + Head head; + struct stat file; + Entry document; + uint64_t index; + int64_t revision; + bool found = false; + *leaves = *small = 0; + bool ok = fd >= 0 && read_head(fd, &file, false, &head, NULL) && map_tree(fd, &file, &head, &tree) && + find(&tree, (Entry){head.catalog, head.catalog_bytes}, key, &index, &found, + &document, &revision) && found && root_height(&tree, document, height) && + (!document.bytes || shape(&tree, document, *height, true, leaves, small)); + if (ok) *file_end = head.end; + release_tree(&tree); + if (fd >= 0) close(fd); + return ok; +} +#endif diff --git a/store/test-tree-store.c b/store/test-tree-store.c new file mode 100644 index 0000000..1130acd --- /dev/null +++ b/store/test-tree-store.c @@ -0,0 +1,1131 @@ +#define _POSIX_C_SOURCE 200809L +#include "store.h" +#include "state.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #x); abort(); } } while (0) + +static int fail_stage = -1; +static int crash; +static int slow_once; /* the next publication pauses before its data sync */ +static int held_sync; /* 1: the next publication waits after its head write; 2: it is waiting */ +static void pause_ms(long ms) { + struct timespec wait = {ms / 1000, (ms % 1000) * 1000000}; + nanosleep(&wait, NULL); +} +int pure_store_test_fault(PureStoreStage stage) { + if (stage == PURE_BEFORE_FILE_SYNC && __atomic_exchange_n(&slow_once, 0, __ATOMIC_SEQ_CST)) pause_ms(100); + int one = 1; + if (stage == PURE_BEFORE_DIR_SYNC && + __atomic_compare_exchange_n(&held_sync, &one, 2, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) + while (__atomic_load_n(&held_sync, __ATOMIC_SEQ_CST) == 2) pause_ms(1); + if ((int)stage != fail_stage) return 0; + if (crash) _exit(77); + return 1; +} +bool pure_tree_test_shape(const char *path, const char *key, unsigned *height, + uint64_t *leaves, uint64_t *small, uint64_t *file_end); + +void pure_draft_test_arena(const PureDraft *draft, size_t *nodes, size_t *bytes); +unsigned long pure_tree_test_publications(void); +unsigned long pure_tree_test_syncs(void); +void pure_tree_test_forget(void); + +/* The file: head slots in blocks 0 and 1 (sequence at +8, end at +16, + * catalog at +24), commits from DATA_START, each a 128-byte record (its head) + * and then its bytes. */ +enum { KEYS = 4, CAPACITY = 1 << 20, SLOT_BLOCK = 4096, DATA_START = 8192, RECORD_SIZE = 128 }; +typedef struct { unsigned char *bytes; size_t size; int64_t revision; } Oracle; + +static uint32_t random_value(uint32_t *state) { + *state ^= *state << 13; + *state ^= *state >> 17; + *state ^= *state << 5; + return *state; +} + +static void expect(const char *path, const char *key, const Oracle *o) { + PureStoredView view; + CHECK(pure_store_get(path, key, PURE_OBJECT_LIMIT, &view) == PURE_OK); + CHECK(view.revision == o->revision && view.body.size == o->size); + CHECK(!o->size || !memcmp(view.body.data, o->bytes, o->size)); + pure_store_release(&view); +} + +static uint64_t file_size(const char *path) { + char name[512]; + snprintf(name, sizeof(name), "%s/pure.tree", path); + struct stat info; + CHECK(stat(name, &info) == 0); + return (uint64_t)info.st_size; +} + +/* Where the newest commit ends. */ +static uint64_t published_end(const char *path, const char *key) { + unsigned height; + uint64_t leaves, small, end; + CHECK(pure_tree_test_shape(path, key, &height, &leaves, &small, &end)); + return end; +} + +static uint64_t field_at(int fd, uint64_t at) { + unsigned char bytes[8]; + CHECK(pread(fd, bytes, 8, (off_t)at) == 8); + uint64_t value = 0; + for (unsigned i = 0; i < 8; ++i) value |= (uint64_t)bytes[i] << (8 * i); + return value; +} + +static uint64_t slot_field(int fd, unsigned slot, unsigned offset) { return field_at(fd, slot * SLOT_BLOCK + offset); } + +/* Where the newest head is: the newer slot's, or the last record after it. */ +static uint64_t newest_head(int fd) { + uint64_t at = slot_field(fd, 0, 8) > slot_field(fd, 1, 8) ? 0 : SLOT_BLOCK; + for (;;) { + uint64_t end = field_at(fd, at + 16); + unsigned char magic[8]; + if (pread(fd, magic, 8, (off_t)end) != 8 || memcmp(magic, "PUREREC1", 8) || + field_at(fd, end + 8) != field_at(fd, at + 8) + 1) return at; + at = end; + } +} + +static void fill(unsigned char *bytes, size_t size, uint32_t *state) { + for (size_t i = 0; i < size; ++i) bytes[i] = (unsigned char)random_value(state); +} + +/* Random edits against a flat oracle, including views held across later + * commits and compactions. Every stored node must stay balanced. */ +static void oracle(const char *path) { + static const char *keys[KEYS] = {"alpha", "b", "gamma-long_key-0123456789", "z"}; + Oracle docs[KEYS] = {{0}}; + unsigned char *insert = malloc(65536); + for (unsigned k = 0; k < KEYS; ++k) CHECK((docs[k].bytes = malloc(CAPACITY))); + CHECK(insert); + uint32_t state = 0x9e3779b9u; + PureStoredView held = {0}; + unsigned char *held_copy = malloc(CAPACITY); + size_t held_size = 0; + unsigned max_height = 0; + size_t max_size = 0, splices = 0; + for (unsigned step = 0; step < 6000; ++step) { + unsigned k = random_value(&state) % KEYS; + Oracle *o = &docs[k]; + uint32_t kind = random_value(&state) % 100; + size_t count = kind < 70 ? random_value(&state) % 24 : + kind < 95 ? random_value(&state) % 6000 : random_value(&state) % 65536; + int64_t revision; + fill(insert, count, &state); + if (kind >= 98 || !o->revision) { + CHECK(pure_store_put(path, keys[k], o->revision, (PureSpan){insert, count}, &revision) == PURE_OK); + memcpy(o->bytes, insert, count); + o->size = count; + } else { + size_t offset = o->size ? random_value(&state) % (o->size + 1) : 0; + size_t remove = o->size - offset ? random_value(&state) % (o->size - offset + 1) : 0; + if (random_value(&state) % 24) remove = remove % 64; + if (o->size - remove + count > CAPACITY) { count = 0; remove = o->size - offset; } + CHECK(pure_store_splice(path, keys[k], o->revision, offset, remove, + (PureSpan){insert, count}, PURE_OBJECT_LIMIT, &revision) == PURE_OK); + memmove(o->bytes + offset + count, o->bytes + offset + remove, o->size - offset - remove); + memcpy(o->bytes + offset, insert, count); + o->size = o->size - remove + count; + ++splices; + } + if (o->size > max_size) max_size = o->size; + CHECK(revision == o->revision + 1); + o->revision = revision; + expect(path, keys[k], o); + unsigned height; + uint64_t leaves, small, end; + CHECK(pure_tree_test_shape(path, keys[k], &height, &leaves, &small, &end)); + CHECK(small == 0); + if (height > max_height) max_height = height; + if (step % 97 == 0) { + if (held.mapped_size) { + CHECK(held.body.size == held_size && !memcmp(held.body.data, held_copy, held_size)); + pure_store_release(&held); + } + CHECK(pure_store_get(path, keys[k], PURE_OBJECT_LIMIT, &held) == PURE_OK); + held_size = held.body.size; + if (held_size) memcpy(held_copy, held.body.data, held_size); + } + } + if (held.mapped_size) { + CHECK(held.body.size == held_size && !memcmp(held.body.data, held_copy, held_size)); + pure_store_release(&held); + } + for (unsigned k = 0; k < KEYS; ++k) { expect(path, keys[k], &docs[k]); free(docs[k].bytes); } + CHECK(max_height >= 2 && max_height <= 4 && max_size > 262144); + printf("{\"oracle_steps\":6000,\"splices\":%zu,\"max_document_bytes\":%zu,\"max_height\":%u}\n", + splices, max_size, max_height); + free(insert); + free(held_copy); +} + +/* A one-byte insert, delete or replacement in a 1 MiB document appends only + * its changed leaf and path, not another copy of the document. */ +static void amplification(const char *path) { + size_t size = 1048576; + unsigned char *bytes = malloc(size + 1); + uint32_t state = 7; + CHECK(bytes); + fill(bytes, size, &state); + int64_t revision; + CHECK(pure_store_put(path, "large", 0, (PureSpan){bytes, size}, &revision) == PURE_OK); + Oracle o = {bytes, size, revision}; + unsigned char byte = 'Q'; + const size_t edits[][3] = {{size / 2, 0, 1}, {size / 3, 1, 0}, {size - 1, 1, 1}, {0, 0, 1}}; + uint64_t worst = 0; + for (size_t i = 0; i < sizeof(edits) / sizeof(*edits); ++i) { + uint64_t before = published_end(path, "large"); + size_t offset = edits[i][0], remove = edits[i][1], count = edits[i][2]; + CHECK(pure_store_splice(path, "large", o.revision, offset, remove, (PureSpan){&byte, count}, + PURE_OBJECT_LIMIT, &revision) == PURE_OK); + memmove(bytes + offset + count, bytes + offset + remove, o.size - offset - remove); + memcpy(bytes + offset, &byte, count); + o.size = o.size - remove + count; + o.revision = revision; + uint64_t after = published_end(path, "large"); + if (after > before && after - before > worst) worst = after - before; + expect(path, "large", &o); + } + /* Compaction may shrink the file; growth per edit is the claim. */ + CHECK(worst > 0 && worst < 16384); + unsigned height; + uint64_t leaves, small, end; + CHECK(pure_tree_test_shape(path, "large", &height, &leaves, &small, &end)); + CHECK(small == 0 && height == 2 && leaves >= 256 && leaves <= 512); + printf("{\"one_byte_edit_max_appended_bytes\":%llu,\"height\":%u,\"leaves\":%llu}\n", + (unsigned long long)worst, height, (unsigned long long)leaves); + free(bytes); +} + +static void faults(const char *path) { + unsigned char text[] = "fault-free"; + int64_t revision; + CHECK(pure_store_put(path, "fault", 0, (PureSpan){text, 10}, &revision) == PURE_OK); + Oracle o = {text, 10, revision}; + uint64_t size = file_size(path), end = published_end(path, "fault"); + for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; ++fail_stage) { + CHECK(pure_store_splice(path, "fault", o.revision, 1, 3, (PureSpan){text, 5}, + PURE_OBJECT_LIMIT, &revision) == PURE_STORE_ERROR); + expect(path, "fault", &o); + /* The bytes stayed unpublished, in the written zeros after the end. */ + CHECK(published_end(path, "fault") == end && file_size(path) == size); + } + fail_stage = PURE_BEFORE_DIR_SYNC; + CHECK(pure_store_splice(path, "fault", o.revision, 10, 0, (PureSpan){text, 2}, + PURE_OBJECT_LIMIT, &revision) == PURE_UNCERTAIN); + fail_stage = -1; + unsigned char next[] = "fault-freefa"; + o = (Oracle){next, 12, o.revision + 1}; + CHECK(revision == o.revision); + expect(path, "fault", &o); +} + +/* A process that dies at each publication stage leaves either the previous + * or the next complete version; the next commit overwrites its unpublished + * bytes. A head it wrote but did not sync is made durable before the next + * writer builds on it. */ +static void crashes(const char *path) { + unsigned char text[] = "crash-base"; + int64_t revision; + CHECK(pure_store_put(path, "crash", 0, (PureSpan){text, 10}, &revision) == PURE_OK); + Oracle o = {text, 10, revision}; + for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { + uint64_t before = published_end(path, "crash"); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + crash = 1; + fail_stage = stage; + pure_store_splice(path, "crash", o.revision, 0, 1, (PureSpan){(const unsigned char *)"C", 1}, + PURE_OBJECT_LIMIT, &revision); + _exit(0); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); + if (stage == PURE_BEFORE_DIR_SYNC) { text[0] = 'C'; ++o.revision; } + expect(path, "crash", &o); + uint64_t published = published_end(path, "crash"); + CHECK(stage == PURE_BEFORE_DIR_SYNC ? published > before : published == before); + unsigned long syncs = pure_tree_test_syncs(); + CHECK(pure_store_splice(path, "crash", o.revision, 10, 0, (PureSpan){(const unsigned char *)"", 0}, + PURE_OBJECT_LIMIT, &revision) == PURE_OK); + o.revision = revision; + expect(path, "crash", &o); + CHECK(pure_tree_test_syncs() - syncs == (stage == PURE_BEFORE_DIR_SYNC ? 2u : 1u)); + CHECK(published_end(path, "crash") > published); + } +} + +static void tear(const char *path, uint64_t at) { + char name[512]; + snprintf(name, sizeof(name), "%s/pure.tree", path); + int fd = open(name, O_RDWR); + unsigned char torn; + CHECK(fd >= 0 && pread(fd, &torn, 1, (off_t)at) == 1); + torn ^= 0x5a; + CHECK(pwrite(fd, &torn, 1, (off_t)at) == 1 && close(fd) == 0); +} + +/* A torn newest record exposes the previous commit to a new process, and the + * next writer overwrites that record and the commit's bytes. A torn slot + * costs only a longer walk: the other slot's head leads along the records. */ +static void torn_head(const char *path) { + unsigned char first[] = "first", second[] = "second"; + int64_t revision; + CHECK(pure_store_put(path, "torn", 0, (PureSpan){first, 5}, &revision) == PURE_OK); + Oracle o = {first, 5, revision}; + uint64_t record = published_end(path, "torn"); /* where the next commit's record goes */ + CHECK(pure_store_put(path, "torn", o.revision, (PureSpan){second, 6}, &revision) == PURE_OK); + tear(path, SLOT_BLOCK + 20); + pure_tree_test_forget(); + expect(path, "torn", &(Oracle){second, 6, revision}); + tear(path, record + 20); + pure_tree_test_forget(); + expect(path, "torn", &o); + CHECK(pure_store_put(path, "torn", o.revision + 1, (PureSpan){first, 1}, &revision) == PURE_CONFLICT); + CHECK(pure_store_put(path, "torn", o.revision, (PureSpan){second, 3}, &revision) == PURE_OK); + o = (Oracle){second, 3, revision}; + CHECK(published_end(path, "torn") < record + RECORD_SIZE + 4096); /* it went where the torn one was */ + pure_tree_test_forget(); + expect(path, "torn", &o); +} + +/* Commits are found along their records: from a slot's head, which every + * CHECKPOINT commits moves on, by a new process too; a damaged record ends + * the walk at the commit before it, which the next writer builds on. */ +static void records(const char *path) { + int64_t revision; + uint64_t ends[41]; + char text[16]; + CHECK(pure_store_put(path, "walk", 0, (PureSpan){(const unsigned char *)"0", 1}, &revision) == PURE_OK); + for (unsigned i = 1; i <= 40; ++i) { + ends[i] = published_end(path, "walk"); + snprintf(text, sizeof(text), "%u", i); + CHECK(pure_store_put(path, "walk", revision, (PureSpan){(const unsigned char *)text, strlen(text)}, + &revision) == PURE_OK); + } + char name[512]; + snprintf(name, sizeof(name), "%s/pure.tree", path); + int fd = open(name, O_RDONLY); + CHECK(fd >= 0); + uint64_t newest_slot = slot_field(fd, 0, 8) > slot_field(fd, 1, 8) ? slot_field(fd, 0, 8) : slot_field(fd, 1, 8); + CHECK(close(fd) == 0); + /* 42 commits (the empty file's first): heads in slots at 17 and 33. */ + CHECK(newest_slot == 33); + pure_tree_test_forget(); + expect(path, "walk", &(Oracle){(unsigned char *)"40", 2, 41}); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + pure_tree_test_forget(); /* a process that never saw them */ + PureStoredView view; + _exit(pure_store_get(path, "walk", 16, &view) == PURE_OK && view.body.size == 2 && + !memcmp(view.body.data, "40", 2) && view.revision == 41 ? 0 : 1); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 0); + /* Commit 37 ("37", revision 38) has a damaged record: the walk ends at 36. */ + tear(path, ends[37] + 30); + pure_tree_test_forget(); + expect(path, "walk", &(Oracle){(unsigned char *)"36", 2, 37}); + CHECK(pure_store_put(path, "walk", 37, (PureSpan){(const unsigned char *)"x", 1}, &revision) == PURE_OK); + CHECK(revision == 38 && published_end(path, "walk") < ends[38]); + pure_tree_test_forget(); + expect(path, "walk", &(Oracle){(unsigned char *)"x", 1, 38}); +} + +/* A reader's view stays byte-identical while another process commits and + * compacts; readers take no lock. */ +static void isolation(const char *path) { + size_t size = 300000; + unsigned char *bytes = malloc(size); + uint32_t state = 99; + CHECK(bytes); + fill(bytes, size, &state); + int64_t revision; + CHECK(pure_store_put(path, "shared", 0, (PureSpan){bytes, size}, &revision) == PURE_OK); + PureStoredView view; + CHECK(pure_store_get(path, "shared", size, &view) == PURE_OK); + uint64_t before = file_size(path); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + int64_t at = revision; + unsigned char byte = 'x'; + for (unsigned i = 0; i < 400; ++i) + if (pure_store_splice(path, "shared", at, (i * 7919u) % size, 1, (PureSpan){&byte, 1}, + PURE_OBJECT_LIMIT, &at) != PURE_OK) _exit(1); + _exit(0); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 0); + CHECK(view.revision == revision && view.body.size == size && !memcmp(view.body.data, bytes, size)); + pure_store_release(&view); + PureStoredView latest; + CHECK(pure_store_get(path, "shared", size, &latest) == PURE_OK && latest.revision == revision + 400); + pure_store_release(&latest); + /* About 2 MB of path copies without compaction would exceed this. */ + CHECK(file_size(path) < before + 3 * 524288); + free(bytes); +} + +/* Damaged branch bytes are reported as storage errors, never trusted. */ +static void corruption(const char *path) { + unsigned char *bytes = malloc(200000); + uint32_t state = 3; + CHECK(bytes); + fill(bytes, 200000, &state); + int64_t revision; + CHECK(pure_store_put(path, "damaged", 0, (PureSpan){bytes, 200000}, &revision) == PURE_OK); + char name[512]; + snprintf(name, sizeof(name), "%s/pure.tree", path); + uint64_t end = published_end(path, "damaged"); + int fd = open(name, O_RDWR); + CHECK(fd >= 0); + unsigned errors = 0; + for (unsigned i = 0; i < 3000; ++i) { + off_t at = (off_t)(DATA_START + random_value(&state) % (end - DATA_START)); + unsigned char original, damaged; + CHECK(pread(fd, &original, 1, at) == 1); + damaged = original ^ (unsigned char)(1u << (random_value(&state) % 8)); + CHECK(pwrite(fd, &damaged, 1, at) == 1); + PureStoredView view; + PureResult result = pure_store_get(path, "damaged", PURE_OBJECT_LIMIT, &view); + CHECK(result == PURE_OK || result == PURE_STORE_ERROR || result == PURE_NOT_FOUND); + if (result == PURE_OK) pure_store_release(&view); else ++errors; + CHECK(pwrite(fd, &original, 1, at) == 1); + } + CHECK(close(fd) == 0 && errors > 0); + free(bytes); +} + +static void many_keys(const char *path) { + int64_t revision; + char key[32]; + for (unsigned i = 0; i < 1024; ++i) { + unsigned id = (i * 617u) % 1024u; + snprintf(key, sizeof(key), "doc-%04u", id); + CHECK(pure_store_put(path, key, 0, (PureSpan){(const unsigned char *)key, strlen(key)}, &revision) == PURE_OK); + } + for (unsigned id = 0; id < 1024; ++id) { + snprintf(key, sizeof(key), "doc-%04u", id); + Oracle o = {(unsigned char *)key, strlen(key), 1}; + expect(path, key, &o); + } + PureStoredView view; + CHECK(pure_store_get(path, "doc-1024", 64, &view) == PURE_NOT_FOUND); + CHECK(pure_store_get(path, "doc-0001", 3, &view) == PURE_BUDGET); + CHECK(pure_store_splice(path, "doc-0002", 1, 9, 0, (PureSpan){0}, PURE_OBJECT_LIMIT, &revision) == PURE_LIMIT); + CHECK(pure_store_put(path, "doc-0003", 0, (PureSpan){0}, &revision) == PURE_CONFLICT && revision == 1); +} + +/* The newest catalog of a one-key store is one leaf; forge its revision. */ +static void revision_limit(const char *path) { + int64_t revision; + CHECK(pure_store_put(path, "work", 0, (PureSpan){(const unsigned char *)"w", 1}, &revision) == PURE_OK); + char name[512]; + snprintf(name, sizeof(name), "%s/pure.tree", path); + int fd = open(name, O_RDWR); + CHECK(fd >= 0); + uint64_t catalog = field_at(fd, newest_head(fd) + 24); + CHECK(catalog >> 63); + const unsigned char maximum[] = {255,255,255,255,255,255,255,127}; + off_t record = (off_t)(catalog & ((UINT64_C(1) << 62) - 1)); + CHECK(pwrite(fd, maximum, 8, record + 80) == 8 && close(fd) == 0); + /* The forged commit no longer matches its digest; this process wrote it + * and does not read it again, a new process would pass over it. */ + PureStoredView view; + CHECK(pure_store_get(path, "work", 1, &view) == PURE_OK && view.revision == INT64_MAX); + pure_store_release(&view); + CHECK(pure_store_splice(path, "work", INT64_MAX, 0, 0, (PureSpan){0}, PURE_OBJECT_LIMIT, &revision) == PURE_LIMIT); +} + +static bool count_key(void *context, const char *key) { + (void)key; + ++*(unsigned *)context; + return true; +} + +static void keys(const char *path) { + unsigned count = 0; + CHECK(pure_store_keys(path, count_key, &count) == PURE_OK && count == 0); + int64_t revision; + const char *names[] = {"m", "a", "zz", "a-b", "A"}; + for (unsigned i = 0; i < 5; ++i) + CHECK(pure_store_put(path, names[i], 0, (PureSpan){0}, &revision) == PURE_OK); + CHECK(pure_store_keys(path, count_key, &count) == PURE_OK && count == 5); +} + +typedef struct { unsigned char *out; size_t spans, largest, limit; } Gather; + +static bool gather(void *context, PureSpan span) { + Gather *g = context; + memcpy(g->out, span.data, span.size); + g->out += span.size; + if (span.size > g->largest) g->largest = span.size; + return ++g->spans != g->limit; +} + +static void expect_draft(const PureDraft *draft, const Oracle *o, unsigned char *scratch) { + Gather g = {scratch, 0, 0, 0}; + CHECK(pure_draft_size(draft) == o->size && pure_draft_revision(draft) == o->revision); + CHECK(pure_draft_read(draft, 0, o->size, gather, &g) == PURE_OK); + CHECK((size_t)(g.out - scratch) == o->size && (!o->size || !memcmp(scratch, o->bytes, o->size))); + CHECK(g.largest <= 4096); +} + +static void apply(Oracle *o, size_t offset, size_t remove, const unsigned char *insert, size_t count) { + memmove(o->bytes + offset + count, o->bytes + offset + remove, o->size - offset - remove); + if (count) memcpy(o->bytes + offset, insert, count); + o->size = o->size - remove + count; +} + +/* Edits live only in the draft until commit; each commit continues from the + * published version with an empty arena. */ +static void drafts(const char *path) { + Oracle o = {malloc(CAPACITY), 0, 0}; + unsigned char *scratch = malloc(CAPACITY), *insert = malloc(65536); + CHECK(o.bytes && scratch && insert); + PureDraft *draft; + PureStoredView view; + CHECK(pure_draft_open(path, "draft", &draft) == PURE_OK); + expect_draft(draft, &o, scratch); + uint32_t state = 0x1234567u; + for (unsigned step = 0; step < 20000; ++step) { + uint32_t kind = random_value(&state) % 100; + size_t count = kind < 80 ? random_value(&state) % 8 : + kind < 98 ? random_value(&state) % 4000 : random_value(&state) % 65536; + size_t offset = o.size ? random_value(&state) % (o.size + 1) : 0; + size_t remove = o.size - offset ? random_value(&state) % (o.size - offset + 1) : 0; + if (random_value(&state) % 64) remove %= 16; + if (o.size - remove + count > CAPACITY) count = 0; + fill(insert, count, &state); + CHECK(pure_draft_splice(draft, offset, remove, (PureSpan){insert, count}) == PURE_OK); + apply(&o, offset, remove, insert, count); + if (step % 97 == 0) expect_draft(draft, &o, scratch); + if (step % 1000 == 999) { + if (o.revision) CHECK(pure_store_get(path, "draft", PURE_OBJECT_LIMIT, &view) == PURE_OK && + view.revision == o.revision); + if (o.revision) pure_store_release(&view); + int64_t revision; + CHECK(pure_draft_commit(draft, &revision) == PURE_OK && revision == o.revision + 1); + o.revision = revision; + expect(path, "draft", &o); + expect_draft(draft, &o, scratch); + size_t nodes, bytes; + pure_draft_test_arena(draft, &nodes, &bytes); + CHECK(nodes == 0); + } + } + CHECK(o.size > 100000); + Gather g = {scratch, 0, 0, 3}; + CHECK(pure_draft_read(draft, 0, o.size, gather, &g) == PURE_OK && g.spans == 3); + CHECK(pure_draft_splice(draft, o.size + 1, 0, (PureSpan){0}) == PURE_LIMIT); + CHECK(pure_draft_splice(draft, 0, o.size + 1, (PureSpan){0}) == PURE_LIMIT); + CHECK(pure_draft_read(draft, o.size, 1, gather, &g) == PURE_LIMIT); + expect_draft(draft, &o, scratch); + pure_draft_close(draft); + printf("{\"draft_steps\":20000,\"draft_commits\":%lld,\"draft_bytes\":%zu}\n", + (long long)o.revision, o.size); + free(o.bytes); + free(scratch); + free(insert); +} + +/* Long typing and deleting at one place keeps only the live path in memory, + * and its commit appends the typed bytes and path, not the document. */ +static void typing(const char *path) { + size_t size = 1048576; + Oracle o = {malloc(size + 40000), size, 0}; + unsigned char *scratch = malloc(size + 40000); + uint32_t state = 5; + CHECK(o.bytes && scratch); + fill(o.bytes, size, &state); + CHECK(pure_store_put(path, "typed", 0, (PureSpan){o.bytes, size}, &o.revision) == PURE_OK); + PureDraft *draft; + CHECK(pure_draft_open(path, "typed", &draft) == PURE_OK); + size_t cursor = size / 2, nodes, bytes, peak = 0; + for (unsigned i = 0; i < 30000; ++i) { + unsigned char byte = (unsigned char)('a' + i % 26); + if (i % 3 == 2) { + CHECK(pure_draft_splice(draft, cursor - 1, 1, (PureSpan){0}) == PURE_OK); + apply(&o, --cursor, 1, NULL, 0); + } else { + CHECK(pure_draft_splice(draft, cursor, 0, (PureSpan){&byte, 1}) == PURE_OK); + apply(&o, cursor++, 0, &byte, 1); + } + pure_draft_test_arena(draft, &nodes, &bytes); + if (bytes > peak) peak = bytes; + } + expect_draft(draft, &o, scratch); + /* Reachable: 10,000 net typed bytes, the leaves they landed in and one + * path. Unreachable copies are reclaimed past max(reachable, 32 KiB). */ + CHECK(peak < 2 * (10000 + 4 * 4096 + 2048) + 32768 + 8192); + uint64_t before = published_end(path, "typed"); + int64_t revision; + CHECK(pure_draft_commit(draft, &revision) == PURE_OK); + o.revision = revision; + uint64_t appended = published_end(path, "typed") - before; + CHECK(appended < 10000 + 16384); + expect(path, "typed", &o); + pure_draft_close(draft); + printf("{\"typing_keystrokes\":30000,\"peak_arena_bytes\":%zu,\"commit_appended_bytes\":%llu}\n", + peak, (unsigned long long)appended); + free(o.bytes); + free(scratch); +} + +/* Two drafts of one key: the second commit conflicts and keeps its edits. */ +static void draft_conflict(const char *path) { + unsigned char scratch[64]; + int64_t revision; + CHECK(pure_store_put(path, "shared", 0, (PureSpan){(const unsigned char *)"base", 4}, &revision) == PURE_OK); + PureDraft *first, *second; + CHECK(pure_draft_open(path, "shared", &first) == PURE_OK && pure_draft_open(path, "shared", &second) == PURE_OK); + CHECK(pure_draft_splice(first, 4, 0, (PureSpan){(const unsigned char *)"-one", 4}) == PURE_OK); + CHECK(pure_draft_splice(second, 0, 0, (PureSpan){(const unsigned char *)"two-", 4}) == PURE_OK); + CHECK(pure_draft_commit(first, &revision) == PURE_OK && revision == 2); + CHECK(pure_draft_commit(second, &revision) == PURE_CONFLICT && revision == 2); + Oracle kept = {(unsigned char *)"two-base", 8, 1}; + expect_draft(second, &kept, scratch); + Oracle stored = {(unsigned char *)"base-one", 8, 2}; + expect(path, "shared", &stored); + pure_draft_close(second); + CHECK(pure_draft_open(path, "shared", &second) == PURE_OK); + expect_draft(second, &stored, scratch); + CHECK(pure_store_put(path, "shared", 2, (PureSpan){(const unsigned char *)"x", 1}, &revision) == PURE_OK); + CHECK(pure_draft_commit(first, &revision) == PURE_CONFLICT && revision == 3); + pure_draft_close(first); + pure_draft_close(second); +} + +/* Another key's writes compact the file under an open draft; the draft's + * stored extents are copied from its old mapping on commit. */ +static void draft_relocation(const char *path) { + unsigned char *big = malloc(300000), scratch[20000]; + uint32_t state = 11; + CHECK(big); + fill(big, 300000, &state); + Oracle o = {malloc(20000), 12000, 0}; + CHECK(o.bytes); + fill(o.bytes, o.size, &state); + CHECK(pure_store_put(path, "kept", 0, (PureSpan){o.bytes, o.size}, &o.revision) == PURE_OK); + PureDraft *draft; + CHECK(pure_draft_open(path, "kept", &draft) == PURE_OK); + CHECK(pure_draft_splice(draft, 6000, 10, (PureSpan){(const unsigned char *)"relocated", 9}) == PURE_OK); + apply(&o, 6000, 10, (const unsigned char *)"relocated", 9); + char name[512]; + snprintf(name, sizeof(name), "%s/pure.tree", path); + struct stat before, after; + CHECK(stat(name, &before) == 0); + int64_t revision = 0; + for (unsigned i = 0; i < 8; ++i) + CHECK(pure_store_put(path, "other", revision, (PureSpan){big, 300000 - i}, &revision) == PURE_OK); + CHECK(stat(name, &after) == 0 && after.st_ino != before.st_ino); + CHECK(pure_draft_commit(draft, &revision) == PURE_OK && revision == o.revision + 1); + o.revision = revision; + expect(path, "kept", &o); + expect_draft(draft, &o, scratch); + Oracle other = {big, 300000 - 7, 8}; + expect(path, "other", &other); + pure_draft_close(draft); + free(o.bytes); + free(big); +} + +/* A held draft keeps earlier versions readable across later edits. */ +static void draft_versions(const char *path) { + static unsigned char bytes[200000], scratch[200000]; + uint32_t state = 17; + fill(bytes, sizeof(bytes), &state); + int64_t revision; + CHECK(pure_store_put(path, "versioned", 0, (PureSpan){bytes, sizeof(bytes)}, &revision) == PURE_OK); + PureDraft *draft; + CHECK(pure_draft_open(path, "versioned", &draft) == PURE_OK); + pure_draft_hold(draft, true); + PureVersion first = pure_draft_version(draft); + for (unsigned i = 0; i < 400; ++i) { + unsigned char byte = (unsigned char)i; + CHECK(pure_draft_splice(draft, (i * 977u) % 150000, 3, (PureSpan){&byte, 1}) == PURE_OK); + } + Gather g = {scratch, 0, 0, 0}; + CHECK(pure_draft_read_version(draft, first, 0, sizeof(bytes), gather, &g) == PURE_OK); + CHECK((size_t)(g.out - scratch) == sizeof(bytes) && !memcmp(scratch, bytes, sizeof(bytes))); + CHECK(pure_draft_size(draft) == sizeof(bytes) - 800); + size_t nodes, held_bytes, after; + pure_draft_test_arena(draft, &nodes, &held_bytes); + pure_draft_hold(draft, false); + pure_draft_test_arena(draft, &nodes, &after); + CHECK(after < held_bytes); + pure_draft_close(draft); +} + +/* A draft without a store: edits, versions, restore and reclamation. */ +static void anonymous_draft(const char *path) { + (void)path; + Oracle o = {malloc(CAPACITY), 0, 0}; + unsigned char *scratch = malloc(CAPACITY), insert[3000]; + PureDraft *draft = pure_draft_new(); + CHECK(draft && o.bytes && scratch && pure_draft_size(draft) == 0 && pure_draft_revision(draft) == 0); + uint32_t state = 42; + for (unsigned step = 0; step < 4000; ++step) { + size_t count = random_value(&state) % (step % 50 ? 12 : sizeof(insert)); + size_t offset = o.size ? random_value(&state) % (o.size + 1) : 0; + size_t remove = o.size - offset ? random_value(&state) % (o.size - offset + 1) % 32 : 0; + fill(insert, count, &state); + bool abandon = random_value(&state) % 5 == 0; + pure_draft_hold(draft, true); + PureVersion before = pure_draft_version(draft); + CHECK(pure_draft_splice(draft, offset, remove, (PureSpan){insert, count}) == PURE_OK); + if (abandon) pure_draft_restore(draft, before); + else apply(&o, offset, remove, insert, count); + pure_draft_hold(draft, false); + if (step % 37 == 0) expect_draft(draft, &o, scratch); + } + expect_draft(draft, &o, scratch); + int64_t revision; + CHECK(pure_draft_commit(draft, &revision) == PURE_INVALID); + size_t nodes, bytes; + pure_draft_test_arena(draft, &nodes, &bytes); + CHECK(bytes < 2 * o.size + 65536); + pure_draft_close(draft); + free(o.bytes); + free(scratch); +} + +/* A failed commit changes neither the store nor the draft. */ +static void draft_faults(const char *path) { + unsigned char scratch[64]; + int64_t revision; + CHECK(pure_store_put(path, "faulty", 0, (PureSpan){(const unsigned char *)"stable", 6}, &revision) == PURE_OK); + PureDraft *draft; + CHECK(pure_draft_open(path, "faulty", &draft) == PURE_OK); + CHECK(pure_draft_splice(draft, 6, 0, (PureSpan){(const unsigned char *)"+edit", 5}) == PURE_OK); + Oracle stored = {(unsigned char *)"stable", 6, 1}, edited = {(unsigned char *)"stable+edit", 11, 1}; + uint64_t size = file_size(path); + for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; ++fail_stage) { + CHECK(pure_draft_commit(draft, &revision) == PURE_STORE_ERROR); + expect(path, "faulty", &stored); + expect_draft(draft, &edited, scratch); + CHECK(file_size(path) == size); + } + fail_stage = PURE_BEFORE_DIR_SYNC; + CHECK(pure_draft_commit(draft, &revision) == PURE_UNCERTAIN && revision == 2); + fail_stage = -1; + expect_draft(draft, &edited, scratch); + CHECK(pure_draft_commit(draft, &revision) == PURE_CONFLICT && revision == 2); + pure_draft_close(draft); + CHECK(pure_draft_open(path, "faulty", &draft) == PURE_OK); + edited.revision = 2; + expect_draft(draft, &edited, scratch); + CHECK(pure_draft_splice(draft, 0, 6, (PureSpan){0}) == PURE_OK); + CHECK(pure_draft_commit(draft, &revision) == PURE_OK && revision == 3); + Oracle final = {(unsigned char *)"+edit", 5, 3}; + expect(path, "faulty", &final); + pure_draft_close(draft); + CHECK(pure_draft_open(path, "bad/key", &draft) == PURE_INVALID && !draft); +} + +static bool list_key(void *context, const char *key) { + char *seen = context; + strcat(seen, key); + strcat(seen, ","); + return true; +} + +/* Several drafts publish in one commit: all or none, under process crashes, + * faults and a stale draft. Runtime values stay out of the document keys. */ +static void batch(const char *path) { + unsigned char scratch[64]; + int64_t revision, revisions[3]; + CHECK(pure_store_put(path, "alpha", 0, (PureSpan){(const unsigned char *)"a0", 2}, &revision) == PURE_OK); + CHECK(pure_store_put(path, "beta", 0, (PureSpan){(const unsigned char *)"b0", 2}, &revision) == PURE_OK); + PureDraft *drafts[3]; + CHECK(pure_draft_open(path, "alpha", &drafts[0]) == PURE_OK && pure_draft_open(path, "beta", &drafts[1]) == PURE_OK); + CHECK(pure_state_open(path, "values", &drafts[2]) == PURE_OK && pure_draft_revision(drafts[2]) == 0); + PureDraft *bad; + CHECK(pure_state_open(path, "bad.name", &bad) == PURE_INVALID && !bad); + CHECK(pure_draft_splice(drafts[0], 2, 0, (PureSpan){(const unsigned char *)"+1", 2}) == PURE_OK); + CHECK(pure_draft_splice(drafts[1], 0, 2, (PureSpan){(const unsigned char *)"B", 1}) == PURE_OK); + CHECK(pure_draft_splice(drafts[2], 0, 0, (PureSpan){(const unsigned char *)"state", 5}) == PURE_OK); + Oracle alpha = {(unsigned char *)"a0", 2, 1}, beta = {(unsigned char *)"b0", 2, 1}; + Oracle alpha_new = {(unsigned char *)"a0+1", 4, 1}, beta_new = {(unsigned char *)"B", 1, 1}; + Oracle state_new = {(unsigned char *)"state", 5, 0}; + /* A process that dies anywhere in the commit leaves all three as before, + * or all three published once the head is written. */ + for (int stage = PURE_AFTER_HEADER; stage <= PURE_BEFORE_DIR_SYNC; ++stage) { + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + crash = 1; + fail_stage = stage; + pure_draft_commit_all(drafts, 3, revisions); + _exit(0); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 77); + PureDraft *state; + CHECK(pure_state_open(path, "values", &state) == PURE_OK); + if (stage < PURE_BEFORE_DIR_SYNC) { + expect(path, "alpha", &alpha); + expect(path, "beta", &beta); + CHECK(pure_draft_revision(state) == 0 && pure_draft_size(state) == 0); + pure_draft_close(state); + continue; + } + Oracle a = alpha_new, b = beta_new, v = state_new; + a.revision = b.revision = 2; + v.revision = 1; + expect(path, "alpha", &a); + expect(path, "beta", &b); + expect_draft(state, &v, scratch); + pure_draft_close(state); + /* Undo the crashed commit's effect for the next checks. */ + CHECK(pure_store_put(path, "alpha", 2, (PureSpan){(const unsigned char *)"a0", 2}, &revision) == PURE_OK); + } + /* The same batch in this process is now stale on every key. */ + CHECK(pure_draft_commit_all(drafts, 3, revisions) == PURE_CONFLICT && revisions[0] == 3); + for (unsigned i = 0; i < 3; ++i) pure_draft_close(drafts[i]); + CHECK(pure_draft_open(path, "alpha", &drafts[0]) == PURE_OK && pure_draft_open(path, "beta", &drafts[1]) == PURE_OK); + CHECK(pure_state_open(path, "values", &drafts[2]) == PURE_OK); + CHECK(pure_draft_splice(drafts[0], 0, 2, (PureSpan){(const unsigned char *)"A", 1}) == PURE_OK); + CHECK(pure_draft_splice(drafts[1], 1, 0, (PureSpan){(const unsigned char *)"!", 1}) == PURE_OK); + CHECK(pure_draft_splice(drafts[2], 5, 0, (PureSpan){(const unsigned char *)"+", 1}) == PURE_OK); + alpha = (Oracle){(unsigned char *)"a0", 2, 3}; + beta = (Oracle){(unsigned char *)"B", 1, 2}; + uint64_t size = file_size(path); + for (fail_stage = PURE_AFTER_HEADER; fail_stage <= PURE_BEFORE_RENAME; ++fail_stage) { + CHECK(pure_draft_commit_all(drafts, 3, revisions) == PURE_STORE_ERROR); + expect(path, "alpha", &alpha); + expect(path, "beta", &beta); + CHECK(file_size(path) == size); + } + fail_stage = -1; + /* One stale draft stops the whole batch. */ + PureDraft *other; + CHECK(pure_draft_open(path, "beta", &other) == PURE_OK); + CHECK(pure_draft_splice(other, 0, 0, (PureSpan){(const unsigned char *)"x", 1}) == PURE_OK); + CHECK(pure_draft_commit(other, &revision) == PURE_OK && revision == 3); + pure_draft_close(other); + CHECK(pure_draft_commit_all(drafts, 3, revisions) == PURE_CONFLICT && revisions[1] == 3); + expect(path, "alpha", &alpha); + pure_draft_close(drafts[1]); + CHECK(pure_draft_open(path, "beta", &drafts[1]) == PURE_OK); + CHECK(pure_draft_splice(drafts[1], 0, 0, (PureSpan){(const unsigned char *)"y", 1}) == PURE_OK); + CHECK(pure_draft_commit_all(drafts, 3, revisions) == PURE_OK); + CHECK(revisions[0] == 4 && revisions[1] == 4 && revisions[2] == 2); + expect(path, "alpha", &(Oracle){(unsigned char *)"A", 1, 4}); + expect(path, "beta", &(Oracle){(unsigned char *)"yxB", 3, 4}); + expect_draft(drafts[2], &(Oracle){(unsigned char *)"state+", 6, 2}, scratch); + /* Drafts continue from the new revisions. */ + CHECK(pure_draft_splice(drafts[0], 1, 0, (PureSpan){(const unsigned char *)"z", 1}) == PURE_OK); + CHECK(pure_draft_commit_all(drafts, 1, revisions) == PURE_OK && revisions[0] == 5); + /* Invalid batches change nothing. */ + PureDraft *twice[2] = {drafts[0], drafts[0]}, *anonymous = pure_draft_new(); + CHECK(pure_draft_commit_all(twice, 2, revisions) == PURE_INVALID); + PureDraft *mixed[2] = {drafts[1], anonymous}; + CHECK(pure_draft_commit_all(mixed, 2, revisions) == PURE_INVALID); + pure_draft_close(anonymous); + for (unsigned i = 0; i < 3; ++i) pure_draft_close(drafts[i]); + /* Runtime values are not documents: listings skip them and document + * operations refuse their names. */ + char seen[256] = ""; + CHECK(pure_store_keys(path, list_key, seen) == PURE_OK && !strcmp(seen, "alpha,beta,")); + CHECK(pure_store_put(path, ".values", 0, (PureSpan){(const unsigned char *)"x", 1}, &revision) != PURE_OK); + /* Many keys, in no particular order, in one commit. */ + PureDraft *many[50]; + int64_t many_revisions[50]; + char key[16]; + for (unsigned i = 0; i < 50; ++i) { + snprintf(key, sizeof(key), "k%02u", (i * 37) % 50); + CHECK((i % 2 ? pure_state_open(path, key, &many[i]) : pure_draft_open(path, key, &many[i])) == PURE_OK); + CHECK(pure_draft_splice(many[i], 0, 0, (PureSpan){(const unsigned char *)key, strlen(key)}) == PURE_OK); + } + CHECK(pure_draft_commit_all(many, 50, many_revisions) == PURE_OK); + for (unsigned i = 0; i < 50; ++i) { + CHECK(many_revisions[i] == 1); + snprintf(key, sizeof(key), "k%02u", (i * 37) % 50); + PureDraft *check; + CHECK((i % 2 ? pure_state_open(path, key, &check) : pure_draft_open(path, key, &check)) == PURE_OK); + expect_draft(check, &(Oracle){(unsigned char *)key, strlen(key), 1}, scratch); + pure_draft_close(check); + pure_draft_close(many[i]); + } +} + +/* A store handle shares one mapping among its drafts, sees commits made by + * other writers when a draft opens, follows a compaction that replaced the + * file, and maps a batch it committed once. */ +static void store_handle(const char *path) { + unsigned char scratch[64]; + int64_t revision, revisions[3]; + CHECK(pure_store_put(path, "shared", 0, (PureSpan){(const unsigned char *)"one", 3}, &revision) == PURE_OK); + PureStore *store; + CHECK(pure_store_open(path, &store) == PURE_OK); + PureDraft *a, *b, *state, *bad; + CHECK(pure_store_draft(store, "shared", &a) == PURE_OK); + expect_draft(a, &(Oracle){(unsigned char *)"one", 3, 1}, scratch); + CHECK(pure_store_draft(store, "bad.key", &bad) == PURE_INVALID && !bad); + /* Another writer commits: the next draft sees it, the open one keeps its version. */ + CHECK(pure_store_put(path, "shared", 1, (PureSpan){(const unsigned char *)"two", 3}, &revision) == PURE_OK); + CHECK(pure_store_draft(store, "shared", &b) == PURE_OK); + expect_draft(b, &(Oracle){(unsigned char *)"two", 3, 2}, scratch); + expect_draft(a, &(Oracle){(unsigned char *)"one", 3, 1}, scratch); + CHECK(pure_draft_splice(a, 3, 0, (PureSpan){(const unsigned char *)"!", 1}) == PURE_OK); + CHECK(pure_draft_commit(a, &revision) == PURE_CONFLICT && revision == 2); + pure_draft_close(a); + /* A batch of handle drafts, a runtime value among them. */ + CHECK(pure_store_state(store, "values", &state) == PURE_OK && pure_draft_revision(state) == 0); + CHECK(pure_store_draft(store, "other", &a) == PURE_OK); + CHECK(pure_draft_splice(b, 3, 0, (PureSpan){(const unsigned char *)"+", 1}) == PURE_OK); + CHECK(pure_draft_splice(state, 0, 0, (PureSpan){(const unsigned char *)"v", 1}) == PURE_OK); + CHECK(pure_draft_splice(a, 0, 0, (PureSpan){(const unsigned char *)"o", 1}) == PURE_OK); + PureDraft *batch[3] = {b, state, a}; + CHECK(pure_draft_commit_all(batch, 3, revisions) == PURE_OK && revisions[0] == 3 && revisions[1] == 1); + expect(path, "shared", &(Oracle){(unsigned char *)"two+", 4, 3}); + expect_draft(state, &(Oracle){(unsigned char *)"v", 1, 1}, scratch); + /* Other writers grow the file past compaction: it is replaced under the handle. */ + unsigned char *big = malloc(100000); + uint32_t seed = 5; + CHECK(big); + fill(big, 100000, &seed); + int64_t filler = 0; + char name[512]; + struct stat before, replaced; + snprintf(name, sizeof(name), "%s/pure.tree", path); + CHECK(stat(name, &before) == 0); + for (int i = 0; i < 8; ++i) CHECK(pure_store_put(path, "filler", filler, (PureSpan){big, 100000}, &filler) == PURE_OK); + CHECK(stat(name, &replaced) == 0 && replaced.st_ino != before.st_ino); + free(big); + PureDraft *after; + CHECK(pure_store_draft(store, "shared", &after) == PURE_OK); + expect_draft(after, &(Oracle){(unsigned char *)"two+", 4, 3}, scratch); + /* Drafts of the old file still read and commit (their extents are copied). */ + expect_draft(b, &(Oracle){(unsigned char *)"two+", 4, 3}, scratch); + CHECK(pure_draft_splice(b, 0, 0, (PureSpan){(const unsigned char *)">", 1}) == PURE_OK); + CHECK(pure_draft_commit(b, &revision) == PURE_OK && revision == 4); + expect(path, "shared", &(Oracle){(unsigned char *)">two+", 5, 4}); + pure_draft_close(after); + pure_draft_close(a); + pure_draft_close(b); + pure_draft_close(state); + pure_store_close(store); +} + +/* Threads of one process commit through their own handles at once: none is + * refused as busy, every commit lands, and edits of one shared key conflict + * instead of overwriting each other. */ +typedef struct { const char *path; unsigned number, conflicts; } Committer; + +static void *commit_many(void *context) { + Committer *c = context; + PureStore *store; + CHECK(pure_store_open(c->path, &store) == PURE_OK); + char key[16]; + snprintf(key, sizeof(key), "t%u", c->number); + for (unsigned i = 0; i < 40; ++i) { + PureDraft *own, *shared; + int64_t revisions[2]; + CHECK(pure_store_draft(store, key, &own) == PURE_OK && pure_store_draft(store, "shared", &shared) == PURE_OK); + CHECK(pure_draft_splice(own, pure_draft_size(own), 0, (PureSpan){(const unsigned char *)"x", 1}) == PURE_OK); + CHECK(pure_draft_splice(shared, pure_draft_size(shared), 0, (PureSpan){(const unsigned char *)"y", 1}) == PURE_OK); + PureDraft *batch[2] = {own, shared}; + PureResult result = pure_draft_commit_all(batch, 2, revisions); + CHECK(result == PURE_OK || result == PURE_CONFLICT); + if (result == PURE_CONFLICT) { + ++c->conflicts; + CHECK(pure_draft_commit(own, &revisions[0]) == PURE_OK); + } + pure_draft_close(own); + pure_draft_close(shared); + } + pure_store_close(store); + return NULL; +} + +static void threads(const char *path) { + pthread_t thread[4]; + Committer committer[4]; + for (unsigned i = 0; i < 4; ++i) { + committer[i] = (Committer){path, i, 0}; + CHECK(pthread_create(&thread[i], NULL, commit_many, &committer[i]) == 0); + } + unsigned conflicts = 0; + for (unsigned i = 0; i < 4; ++i) { + CHECK(pthread_join(thread[i], NULL) == 0); + conflicts += committer[i].conflicts; + } + PureStoredView view; + char key[16]; + for (unsigned i = 0; i < 4; ++i) { + snprintf(key, sizeof(key), "t%u", i); + CHECK(pure_store_get(path, key, PURE_OBJECT_LIMIT, &view) == PURE_OK && view.body.size == 40 && view.revision == 40); + pure_store_release(&view); + } + CHECK(pure_store_get(path, "shared", PURE_OBJECT_LIMIT, &view) == PURE_OK); + CHECK(view.body.size == 160 - conflicts && view.revision == (int64_t)(160 - conflicts)); + pure_store_release(&view); +} + +/* Commits that arrive while another is being published share the next + * publication: one head for all of them. */ +typedef struct { const char *path; unsigned number; PureResult result; } Single; + +static void *commit_one(void *context) { + Single *s = context; + PureStore *store; + PureDraft *draft; + char key[16]; + int64_t revision; + snprintf(key, sizeof(key), "g%u", s->number); + CHECK(pure_store_open(s->path, &store) == PURE_OK && pure_store_draft(store, key, &draft) == PURE_OK); + CHECK(pure_draft_splice(draft, 0, 0, (PureSpan){(const unsigned char *)key, strlen(key)}) == PURE_OK); + s->result = pure_draft_commit(draft, &revision); + CHECK(s->result != PURE_OK || revision == 1); + pure_draft_close(draft); + pure_store_close(store); + return NULL; +} + +static void grouped(const char *path) { + pthread_t thread[5]; + Single single[5]; + unsigned long before = pure_tree_test_publications(); + __atomic_store_n(&slow_once, 1, __ATOMIC_SEQ_CST); + for (unsigned i = 0; i < 5; ++i) { + single[i] = (Single){path, i, PURE_INVALID}; + CHECK(pthread_create(&thread[i], NULL, commit_one, &single[i]) == 0); + if (!i) pause_ms(20); /* the first is now publishing, slowly */ + } + for (unsigned i = 0; i < 5; ++i) CHECK(pthread_join(thread[i], NULL) == 0 && single[i].result == PURE_OK); + CHECK(pure_tree_test_publications() - before == 2); + char key[16]; + for (unsigned i = 0; i < 5; ++i) { + snprintf(key, sizeof(key), "g%u", i); + expect(path, key, &(Oracle){(unsigned char *)key, strlen(key), 1}); + } +} + +/* A crash during a commit's only sync can leave its head on the device + * without all of its bytes. A process that starts after it passes over that + * head to the previous commit, and the next writer overwrites both. */ +static void lost_commit(const char *path) { + unsigned char first[] = "first", second[] = "second"; + int64_t revision; + CHECK(pure_store_put(path, "lost", 0, (PureSpan){first, 5}, &revision) == PURE_OK); + Oracle o = {first, 5, revision}; + uint64_t start = published_end(path, "lost"); + CHECK(pure_store_put(path, "lost", o.revision, (PureSpan){second, 6}, &revision) == PURE_OK); + uint64_t end = published_end(path, "lost"); + char name[512]; + snprintf(name, sizeof(name), "%s/pure.tree", path); + int fd = open(name, O_RDWR); + off_t at = (off_t)(start + (end - start) / 2); + unsigned char byte; + CHECK(fd >= 0 && pread(fd, &byte, 1, at) == 1); + byte ^= 0xff; + CHECK(pwrite(fd, &byte, 1, at) == 1 && close(fd) == 0); + pure_tree_test_forget(); + expect(path, "lost", &o); + PureStore *store; + PureDraft *draft; + CHECK(pure_store_open(path, &store) == PURE_OK && pure_store_draft(store, "lost", &draft) == PURE_OK); + CHECK(pure_draft_revision(draft) == o.revision); + pure_draft_close(draft); + CHECK(pure_store_put(path, "lost", o.revision + 1, (PureSpan){first, 1}, &revision) == PURE_CONFLICT); + CHECK(pure_store_put(path, "lost", o.revision, (PureSpan){second, 3}, &revision) == PURE_OK); + o = (Oracle){second, 3, revision}; + CHECK(published_end(path, "lost") < end); /* it started where the lost commit did */ + CHECK(pure_store_draft(store, "lost", &draft) == PURE_OK && pure_draft_revision(draft) == o.revision); + pure_draft_close(draft); + pure_store_close(store); + pure_tree_test_forget(); + expect(path, "lost", &o); +} + +/* While a commit's sync is pending, this process keeps reading the previous + * version: a crash could still take the new one back. */ +static void *commit_held(void *context) { + int64_t revision; + CHECK(pure_store_put(context, "pending", 1, (PureSpan){(const unsigned char *)"new", 3}, &revision) == PURE_OK); + return NULL; +} + +static void pending_head(const char *path) { + int64_t revision; + CHECK(pure_store_put(path, "pending", 0, (PureSpan){(const unsigned char *)"old", 3}, &revision) == PURE_OK); + PureStore *store; + PureDraft *draft; + CHECK(pure_store_open(path, &store) == PURE_OK); + __atomic_store_n(&held_sync, 1, __ATOMIC_SEQ_CST); + pthread_t writer; + CHECK(pthread_create(&writer, NULL, commit_held, (void *)path) == 0); + while (__atomic_load_n(&held_sync, __ATOMIC_SEQ_CST) != 2) pause_ms(1); + CHECK(pure_store_draft(store, "pending", &draft) == PURE_OK && pure_draft_revision(draft) == 1); + pure_draft_close(draft); + expect(path, "pending", &(Oracle){(unsigned char *)"old", 3, 1}); + __atomic_store_n(&held_sync, 0, __ATOMIC_SEQ_CST); + CHECK(pthread_join(writer, NULL) == 0); + CHECK(pure_store_draft(store, "pending", &draft) == PURE_OK && pure_draft_revision(draft) == 2); + pure_draft_close(draft); + pure_store_close(store); + expect(path, "pending", &(Oracle){(unsigned char *)"new", 3, 2}); +} + +/* A commit costs one sync, and commits into the written zeros after the end + * leave the file size alone (no journal commit for a new size). */ +static void one_sync(const char *path) { + int64_t revision; + CHECK(pure_store_put(path, "sync", 0, (PureSpan){(const unsigned char *)"0", 1}, &revision) == PURE_OK); + uint64_t size = file_size(path); + unsigned long syncs = pure_tree_test_syncs(); + for (unsigned i = 0; i < 20; ++i) + CHECK(pure_store_put(path, "sync", revision, (PureSpan){(const unsigned char *)"x", 1}, &revision) == PURE_OK); + CHECK(pure_tree_test_syncs() - syncs == 20 && file_size(path) == size); +} + +static void clean(const char *path) { + DIR *entries = opendir(path); + CHECK(entries); + struct dirent *entry; + while ((entry = readdir(entries))) { + if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue; + CHECK(strncmp(entry->d_name, ".pending-", 9)); + CHECK(unlinkat(dirfd(entries), entry->d_name, 0) == 0); + } + CHECK(closedir(entries) == 0 && rmdir(path) == 0); +} + +int main(void) { + void (*checks[])(const char *) = {oracle, amplification, faults, crashes, torn_head, + isolation, corruption, many_keys, revision_limit, keys, + drafts, typing, draft_conflict, draft_relocation, draft_faults, + draft_versions, anonymous_draft, batch, store_handle, threads, grouped, + lost_commit, pending_head, one_sync, records}; + for (size_t i = 0; i < sizeof(checks) / sizeof(*checks); ++i) { + char path[] = "/tmp/pure-tree-check-XXXXXX"; + CHECK(mkdtemp(path) && pure_store_init(path) == PURE_OK); + checks[i](path); + CHECK(pure_store_init(path) == PURE_OK); + clean(path); + } + puts("{\"tree_store\":\"random-oracle-balance-held-views-one-byte-amplification-faults-" + "process-crash-torn-head-lockless-isolation-compaction-corruption-1024-keys-revision-limit-key-listing-" + "draft-oracle-typing-arena-conflict-relocation-commit-faults-held-versions-anonymous-" + "atomic-batch-runtime-keys-store-handle-threads-group-commit-lost-commit-pending-head-one-sync-records\"}"); + return 0; +} -- cgit v1.2.3