Write each commit's head just before its bytes: one range per sync (081)

A tree store commit wrote its bytes over the written zeros of the tail
and its head into a slot at the start of the file: two ranges, and on
the Pi's SD card a sync of two ranges costs about twice one (2.4 ms
against 1.2 ms). A commit now writes a 128-byte record (its head) at
the published end and its bytes right after it, then syncs once. Every
16 commits the head also goes into the older slot. The newest commit
is found from the newer slot's head, or from the newest head this
process has seen in the file while its record is still there, along
the records whose bytes are whole; a torn record or bytes end the walk
at the commit before, which the next writer overwrites. Slots of this
format are PURETRE3; files of the earlier one are read and continued.

Store commits went from 220-233/s to 371-396/s (p50 2.5 to 1.3 ms),
and the scoreboard's durable posts from 13-16k/s to 21-26k/s with 64
connections and from 2.2-2.4k/s to 2.9-3.9k/s with 8, measured
alternately with the previous build. The torn head check now tears a
slot (the walk goes on from the other) and the newest record (a new
process sees the commit before); a new check walks 42 commits from the
slots, in a fresh process too, and stops at a damaged record.

3 files changed, 324 insertions, 0 deletions

diff --git a/research/experiments/012-tail-records/2026-09-26-ext4-linux-aarch64.json b/research/experiments/012-tail-records/2026-09-26-ext4-linux-aarch64.json
new file mode 100644
index 0000000..457111d
--- /dev/null
+++ b/research/experiments/012-tail-records/2026-09-26-ext4-linux-aarch64.json
@@ -0,0 +1,184 @@
+{
+ "rounds": [
+  {
+   "probe": [
+    {
+     "way": "bytes, head in a slot at the start, fdatasync",
+     "bytes": 2048,
+     "p50_ms": 2.4,
+     "p90_ms": 3.11,
+     "max_ms": 62.79
+    },
+    {
+     "way": "head just before the bytes, fdatasync",
+     "bytes": 2048,
+     "p50_ms": 1.23,
+     "p90_ms": 3.85,
+     "max_ms": 12.27
+    },
+    {
+     "way": "bytes only, fdatasync",
+     "bytes": 2048,
+     "p50_ms": 1.23,
+     "p90_ms": 3.86,
+     "max_ms": 12.2
+    },
+    {
+     "way": "bytes, head in a slot at the start, fdatasync",
+     "bytes": 8192,
+     "p50_ms": 2.41,
+     "p90_ms": 4.51,
+     "max_ms": 24.86
+    },
+    {
+     "way": "head just before the bytes, fdatasync",
+     "bytes": 8192,
+     "p50_ms": 1.57,
+     "p90_ms": 4.36,
+     "max_ms": 56.75
+    },
+    {
+     "way": "bytes only, fdatasync",
+     "bytes": 8192,
+     "p50_ms": 1.18,
+     "p90_ms": 1.58,
+     "max_ms": 1.7
+    }
+   ],
+   "commits": {
+    "heads in slots": {
+     "commits_per_second": 220,
+     "p50_ms": 2.48,
+     "p90_ms": 5.99,
+     "p99_ms": 59.68
+    },
+    "heads as records": {
+     "commits_per_second": 371,
+     "p50_ms": 1.29,
+     "p90_ms": 3.93,
+     "p99_ms": 52.98
+    }
+   }
+  },
+  {
+   "probe": [
+    {
+     "way": "bytes, head in a slot at the start, fdatasync",
+     "bytes": 2048,
+     "p50_ms": 2.44,
+     "p90_ms": 5.4,
+     "max_ms": 13.3
+    },
+    {
+     "way": "head just before the bytes, fdatasync",
+     "bytes": 2048,
+     "p50_ms": 1.19,
+     "p90_ms": 1.76,
+     "max_ms": 12.16
+    },
+    {
+     "way": "bytes only, fdatasync",
+     "bytes": 2048,
+     "p50_ms": 1.2,
+     "p90_ms": 1.56,
+     "max_ms": 15.32
+    },
+    {
+     "way": "bytes, head in a slot at the start, fdatasync",
+     "bytes": 8192,
+     "p50_ms": 2.71,
+     "p90_ms": 8.51,
+     "max_ms": 23.39
+    },
+    {
+     "way": "head just before the bytes, fdatasync",
+     "bytes": 8192,
+     "p50_ms": 1.18,
+     "p90_ms": 1.6,
+     "max_ms": 32.89
+    },
+    {
+     "way": "bytes only, fdatasync",
+     "bytes": 8192,
+     "p50_ms": 1.21,
+     "p90_ms": 3.9,
+     "max_ms": 74.29
+    }
+   ],
+   "commits": {
+    "heads as records": {
+     "commits_per_second": 372,
+     "p50_ms": 1.29,
+     "p90_ms": 4.04,
+     "p99_ms": 30.55
+    },
+    "heads in slots": {
+     "commits_per_second": 229,
+     "p50_ms": 2.52,
+     "p90_ms": 5.54,
+     "p99_ms": 61.86
+    }
+   }
+  },
+  {
+   "probe": [
+    {
+     "way": "bytes, head in a slot at the start, fdatasync",
+     "bytes": 2048,
+     "p50_ms": 2.44,
+     "p90_ms": 3.94,
+     "max_ms": 53.75
+    },
+    {
+     "way": "head just before the bytes, fdatasync",
+     "bytes": 2048,
+     "p50_ms": 1.21,
+     "p90_ms": 1.85,
+     "max_ms": 4.43
+    },
+    {
+     "way": "bytes only, fdatasync",
+     "bytes": 2048,
+     "p50_ms": 1.21,
+     "p90_ms": 1.57,
+     "max_ms": 4.0
+    },
+    {
+     "way": "bytes, head in a slot at the start, fdatasync",
+     "bytes": 8192,
+     "p50_ms": 2.4,
+     "p90_ms": 3.15,
+     "max_ms": 36.75
+    },
+    {
+     "way": "head just before the bytes, fdatasync",
+     "bytes": 8192,
+     "p50_ms": 1.21,
+     "p90_ms": 4.48,
+     "max_ms": 33.15
+    },
+    {
+     "way": "bytes only, fdatasync",
+     "bytes": 8192,
+     "p50_ms": 1.12,
+     "p90_ms": 1.56,
+     "max_ms": 32.79
+    }
+   ],
+   "commits": {
+    "heads in slots": {
+     "commits_per_second": 233,
+     "p50_ms": 2.55,
+     "p90_ms": 6.04,
+     "p99_ms": 34.19
+    },
+    "heads as records": {
+     "commits_per_second": 396,
+     "p50_ms": 1.26,
+     "p90_ms": 3.9,
+     "p99_ms": 28.43
+    }
+   }
+  }
+ ]
+}
diff --git a/research/experiments/012-tail-records/probe.c b/research/experiments/012-tail-records/probe.c
new file mode 100644
index 0000000..83c1ed6
--- /dev/null
+++ b/research/experiments/012-tail-records/probe.c
@@ -0,0 +1,71 @@
+#define _GNU_SOURCE
+/* What a small commit's sync costs by where its head goes, on one device:
+ * a commit of SIZE bytes written over zeros written earlier (no new size or
+ * blocks, so no journal commit), 40 times per way. The ways run round-robin
+ * so device drift reaches all of them alike.
+ * probe DIRECTORY [SIZE] -> one JSON object per way (milliseconds). */
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+static double now_ms(void) {
+    struct timespec t;
+    clock_gettime(CLOCK_MONOTONIC, &t);
+    return t.tv_sec * 1e3 + t.tv_nsec / 1e6;
+}
+static int order(const void *a, const void *b) {
+    double x = *(const double *)a, y = *(const double *)b;
+    return x < y ? -1 : x > y;
+}
+enum { WAYS = 3, ROUNDS = 40, HEAD = 128, DATA = 8192 };
+static const char *names[WAYS] = {
+    "bytes, head in a slot at the start, fdatasync",
+    "head just before the bytes, fdatasync",
+    "bytes only, fdatasync",
+};
+int main(int argc, char **argv) {
+    const char *directory = argc > 1 ? argv[1] : "/var/tmp";
+    size_t size = argc > 2 ? (size_t)atoi(argv[2]) : 8192;
+    char path[WAYS][512];
+    int fd[WAYS];
+    off_t end[WAYS];
+    char *bytes = malloc(size), head[HEAD];
+    if (!bytes) return 1;
+    memset(bytes, 'r', size);
+    for (int w = 0; w < WAYS; ++w) {
+        snprintf(path[w], sizeof(path[w]), "%s/pure-record-probe-%d", directory, w);
+        fd[w] = open(path[w], O_RDWR | O_CREAT | O_TRUNC, 0600);
+        if (fd[w] < 0) return 1;
+        size_t zeros_size = DATA + (size_t)(ROUNDS + 2) * (size + HEAD);
+        char *zeros = calloc(1, zeros_size);
+        if (!zeros || pwrite(fd[w], zeros, zeros_size, 0) != (ssize_t)zeros_size) return 1;
+        free(zeros);
+        fsync(fd[w]);
+        end[w] = DATA;
+    }
+    double t[WAYS][ROUNDS];
+    for (int r = 0; r < ROUNDS; ++r)
+        for (int w = 0; w < WAYS; ++w) {
+            memset(head, 'h' + r % 2, sizeof(head));
+            double start = now_ms();
+            if (w == 1 && pwrite(fd[w], head, HEAD, end[w]) != HEAD) return 1;
+            off_t at = end[w] + (w == 1 ? HEAD : 0);
+            if (pwrite(fd[w], bytes, size, at) != (ssize_t)size) return 1;
+            end[w] = at + (off_t)size;
+            if (w == 0 && pwrite(fd[w], head, HEAD, (r % 2) * 4096) != HEAD) return 1;
+            fdatasync(fd[w]);
+            t[w][r] = now_ms() - start;
+        }
+    for (int w = 0; w < WAYS; ++w) {
+        qsort(t[w], ROUNDS, sizeof(double), order);
+        printf("{\"way\":\"%s\",\"bytes\":%zu,\"p50_ms\":%.2f,\"p90_ms\":%.2f,\"max_ms\":%.2f}\n", names[w], size,
+               t[w][ROUNDS / 2], t[w][ROUNDS * 9 / 10], t[w][ROUNDS - 1]);
+        close(fd[w]);
+        unlink(path[w]);
+    }
+    free(bytes);
+    return 0;
+}
diff --git a/research/experiments/012-tail-records/run.py b/research/experiments/012-tail-records/run.py
new file mode 100755
index 0000000..7fc342e
--- /dev/null
+++ b/research/experiments/012-tail-records/run.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Commit cost by where the head goes (see docs/research/081-tail-records.md).
+
+Runs probe.c (a small commit's sync with its head in a slot at the start, or
+just before its bytes) for 2 KiB and 8 KiB commits, and ../010-one-sync-commit
+/commits.c (tree store commit latency) built against the store of --before
+(heads in slots) and the current one (heads as records before the bytes),
+alternating their order per round so device drift reaches both alike. Prints
+one JSON document.
+"""
+import argparse
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[2]
+PURE = ROOT / 'prototypes' / 'connections'
+COMMITS = HERE.parent / '010-one-sync-commit' / 'commits.c'
+
+
+def build(output, sources):
+    subprocess.run(['cc', '-O2', '-std=c17', f'-I{PURE}', '-o', str(output)] + [str(s) for s in sources] + ['-lpthread'],
+                   check=True)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--directory', type=Path, default=Path('/var/tmp'), help='on the device to measure')
+    parser.add_argument('--before', default='2ef0d40', help='a revision with heads in slots')
+    parser.add_argument('--rounds', type=int, default=3)
+    parser.add_argument('--commits', type=int, default=400)
+    args = parser.parse_args()
+    with tempfile.TemporaryDirectory() as work:
+        work = Path(work)
+        before = work / 'store-tree-before.c'
+        before.write_text(subprocess.run(['git', '-C', str(ROOT), 'show', f'{args.before}:prototypes/connections/store-tree.c'],
+                                         capture_output=True, text=True, check=True).stdout)
+        build(work / 'probe', [HERE / 'probe.c'])
+        builds = {'heads in slots': work / 'commits-before', 'heads as records': work / 'commits-after'}
+        build(builds['heads in slots'], [COMMITS, before])
+        build(builds['heads as records'], [COMMITS, PURE / 'store-tree.c'])
+        results = {'rounds': []}
+        for round_ in range(args.rounds):
+            measured = {'probe': [], 'commits': {}}
+            for size in (2048, 8192):
+                probe = subprocess.run([str(work / 'probe'), str(args.directory), str(size)], capture_output=True,
+                                       text=True, check=True)
+                measured['probe'] += [json.loads(line) for line in probe.stdout.splitlines()]
+            names = list(builds) if round_ % 2 == 0 else list(builds)[::-1]
+            for name in names:
+                store = Path(tempfile.mkdtemp(dir=args.directory, prefix='pure-commit-cost-'))
+                shutil.rmtree(store)
+                try:
+                    run = subprocess.run([str(builds[name]), str(store), str(args.commits)], capture_output=True, text=True,
+                                         check=True)
+                    measured['commits'][name] = json.loads(run.stdout)
+                finally:
+                    shutil.rmtree(store, ignore_errors=True)
+            results['rounds'].append(measured)
+            print(json.dumps(measured), file=sys.stderr, flush=True)
+        print(json.dumps(results, indent=1))
+
+
+if __name__ == '__main__':
+    sys.exit(main())