Give a compacted file the zeros of its whole cycle (085)

Commits grew the tree store's tail 256 KiB at a time, each growth a sync
that also committed the ext4 journal for the new size and blocks. A file
installed by compaction now carries written zeros up to where the next
compaction will come (4 x live + slack, plus a commit's margin), capped at
the size of the file it replaces, so a store never takes more space than
before and shrinks when its live data does. The cycle's commits then only
overwrite blocks the file has.

Store commits: 317.5 -> 385.5 a second, ext4 journal commits 109 -> 45 for
4,000 commits; durable app posts +5.3%, their p99 the same within this SD
card's noise. The new cycles test keeps sizes still between compactions
once the live data is whole, and shows the file shrinking with it.

5 files changed, 149 insertions, 3 deletions

diff --git a/research/experiments/013-parallel-syncs/2026-09-27-journal-before-085-ext4-linux-aarch64.jsonl b/research/experiments/013-parallel-syncs/2026-09-27-journal-before-085-ext4-linux-aarch64.jsonl
new file mode 100644
index 0000000..5b8320d
--- /dev/null
+++ b/research/experiments/013-parallel-syncs/2026-09-27-journal-before-085-ext4-linux-aarch64.jsonl
@@ -0,0 +1,20 @@
+{"round": 0, "store": "as it is", "commits_per_second": 362, "p50_ms": 1.26, "p90_ms": 3.98, "p99_ms": 55.11, "journal_commits": 113}
+{"round": 0, "store": "no compaction", "commits_per_second": 427, "p50_ms": 1.27, "p90_ms": 3.91, "p99_ms": 36.68, "journal_commits": 44}
+{"round": 0, "store": "no compaction, no growth", "commits_per_second": 541, "p50_ms": 1.27, "p90_ms": 3.17, "p99_ms": 5.49, "journal_commits": 8}
+{"round": 0, "store": "compacted file sized like the old", "commits_per_second": 477, "p50_ms": 1.26, "p90_ms": 3.21, "p99_ms": 10.87, "journal_commits": 46}
+{"round": 1, "store": "no compaction", "commits_per_second": 458, "p50_ms": 1.28, "p90_ms": 3.52, "p99_ms": 16.56, "journal_commits": 43}
+{"round": 1, "store": "no compaction, no growth", "commits_per_second": 440, "p50_ms": 1.27, "p90_ms": 4.08, "p99_ms": 25.17, "journal_commits": 7}
+{"round": 1, "store": "compacted file sized like the old", "commits_per_second": 413, "p50_ms": 1.26, "p90_ms": 3.22, "p99_ms": 25.22, "journal_commits": 48}
+{"round": 1, "store": "as it is", "commits_per_second": 370, "p50_ms": 1.29, "p90_ms": 4.01, "p99_ms": 33.16, "journal_commits": 110}
+{"round": 2, "store": "no compaction, no growth", "commits_per_second": 424, "p50_ms": 1.28, "p90_ms": 3.94, "p99_ms": 25.66, "journal_commits": 9}
+{"round": 2, "store": "compacted file sized like the old", "commits_per_second": 461, "p50_ms": 1.26, "p90_ms": 3.3, "p99_ms": 21.71, "journal_commits": 46}
+{"round": 2, "store": "as it is", "commits_per_second": 340, "p50_ms": 1.28, "p90_ms": 4.21, "p99_ms": 35.79, "journal_commits": 114}
+{"round": 2, "store": "no compaction", "commits_per_second": 404, "p50_ms": 1.28, "p90_ms": 4.13, "p99_ms": 31.76, "journal_commits": 33}
+{"round": 3, "store": "compacted file sized like the old", "commits_per_second": 433, "p50_ms": 1.25, "p90_ms": 3.13, "p99_ms": 25.18, "journal_commits": 46}
+{"round": 3, "store": "as it is", "commits_per_second": 373, "p50_ms": 1.27, "p90_ms": 3.94, "p99_ms": 56.67, "journal_commits": 114}
+{"round": 3, "store": "no compaction", "commits_per_second": 384, "p50_ms": 1.27, "p90_ms": 3.98, "p99_ms": 53.28, "journal_commits": 36}
+{"round": 3, "store": "no compaction, no growth", "commits_per_second": 475, "p50_ms": 1.27, "p90_ms": 3.91, "p99_ms": 21.46, "journal_commits": 13}
+{"store": "as it is", "commits": 4000, "commits_per_second": 366.0, "p50_ms": 1.275, "p90_ms": 3.995, "p99_ms": 45.45, "journal_commits": 113.5}
+{"store": "no compaction", "commits": 4000, "commits_per_second": 415.5, "p50_ms": 1.275, "p90_ms": 3.9450000000000003, "p99_ms": 34.22, "journal_commits": 39.5}
+{"store": "no compaction, no growth", "commits": 4000, "commits_per_second": 457.5, "p50_ms": 1.27, "p90_ms": 3.925, "p99_ms": 23.315, "journal_commits": 8.5}
+{"store": "compacted file sized like the old", "commits": 4000, "commits_per_second": 447.0, "p50_ms": 1.26, "p90_ms": 3.215, "p99_ms": 23.445, "journal_commits": 46.0}
diff --git a/research/experiments/013-parallel-syncs/2026-09-27-journal-ext4-linux-aarch64.jsonl b/research/experiments/013-parallel-syncs/2026-09-27-journal-ext4-linux-aarch64.jsonl
new file mode 100644
index 0000000..0c14f32
--- /dev/null
+++ b/research/experiments/013-parallel-syncs/2026-09-27-journal-ext4-linux-aarch64.jsonl
@@ -0,0 +1,20 @@
+{"round": 0, "store": "as it is", "commits_per_second": 411, "p50_ms": 1.27, "p90_ms": 3.96, "p99_ms": 25.68, "journal_commits": 52}
+{"round": 0, "store": "no compaction", "commits_per_second": 449, "p50_ms": 1.27, "p90_ms": 3.99, "p99_ms": 23.52, "journal_commits": 46}
+{"round": 0, "store": "no compaction, no growth", "commits_per_second": 519, "p50_ms": 1.27, "p90_ms": 3.14, "p99_ms": 5.55, "journal_commits": 4}
+{"round": 0, "store": "tail grown as commits need it", "commits_per_second": 346, "p50_ms": 1.29, "p90_ms": 4.01, "p99_ms": 26.25, "journal_commits": 117}
+{"round": 1, "store": "no compaction", "commits_per_second": 360, "p50_ms": 1.28, "p90_ms": 3.94, "p99_ms": 32.97, "journal_commits": 42}
+{"round": 1, "store": "no compaction, no growth", "commits_per_second": 413, "p50_ms": 1.28, "p90_ms": 4.07, "p99_ms": 26.58, "journal_commits": 18}
+{"round": 1, "store": "tail grown as commits need it", "commits_per_second": 270, "p50_ms": 1.28, "p90_ms": 4.34, "p99_ms": 50.93, "journal_commits": 110}
+{"round": 1, "store": "as it is", "commits_per_second": 382, "p50_ms": 1.25, "p90_ms": 3.99, "p99_ms": 26.49, "journal_commits": 40}
+{"round": 2, "store": "no compaction, no growth", "commits_per_second": 463, "p50_ms": 1.27, "p90_ms": 3.96, "p99_ms": 22.29, "journal_commits": 21}
+{"round": 2, "store": "tail grown as commits need it", "commits_per_second": 289, "p50_ms": 1.27, "p90_ms": 3.99, "p99_ms": 32.56, "journal_commits": 104}
+{"round": 2, "store": "as it is", "commits_per_second": 356, "p50_ms": 1.27, "p90_ms": 3.93, "p99_ms": 37.21, "journal_commits": 46}
+{"round": 2, "store": "no compaction", "commits_per_second": 393, "p50_ms": 1.26, "p90_ms": 4.08, "p99_ms": 32.96, "journal_commits": 41}
+{"round": 3, "store": "tail grown as commits need it", "commits_per_second": 406, "p50_ms": 1.27, "p90_ms": 4.12, "p99_ms": 26.45, "journal_commits": 108}
+{"round": 3, "store": "as it is", "commits_per_second": 389, "p50_ms": 1.27, "p90_ms": 4.06, "p99_ms": 27.31, "journal_commits": 44}
+{"round": 3, "store": "no compaction", "commits_per_second": 392, "p50_ms": 1.26, "p90_ms": 3.4, "p99_ms": 52.8, "journal_commits": 44}
+{"round": 3, "store": "no compaction, no growth", "commits_per_second": 469, "p50_ms": 1.27, "p90_ms": 3.88, "p99_ms": 22.84, "journal_commits": 15}
+{"store": "as it is", "commits": 4000, "commits_per_second": 385.5, "p50_ms": 1.27, "p90_ms": 3.975, "p99_ms": 26.9, "journal_commits": 45.0}
+{"store": "no compaction", "commits": 4000, "commits_per_second": 392.5, "p50_ms": 1.2650000000000001, "p90_ms": 3.965, "p99_ms": 32.965, "journal_commits": 43.0}
+{"store": "no compaction, no growth", "commits": 4000, "commits_per_second": 466.0, "p50_ms": 1.27, "p90_ms": 3.92, "p99_ms": 22.564999999999998, "journal_commits": 16.5}
+{"store": "tail grown as commits need it", "commits": 4000, "commits_per_second": 317.5, "p50_ms": 1.275, "p90_ms": 4.0649999999999995, "p99_ms": 29.505000000000003, "journal_commits": 109.0}
diff --git a/research/experiments/013-parallel-syncs/2026-09-27-store-variants-085-ext4-linux-aarch64.jsonl b/research/experiments/013-parallel-syncs/2026-09-27-store-variants-085-ext4-linux-aarch64.jsonl
new file mode 100644
index 0000000..5e26b0f
--- /dev/null
+++ b/research/experiments/013-parallel-syncs/2026-09-27-store-variants-085-ext4-linux-aarch64.jsonl
@@ -0,0 +1,36 @@
+{"round": 0, "store": "as it is", "posts_per_second": 23632.25, "p50_us": 1984, "p99_us": 21504, "max_us": 61440, "errors": 0}
+{"round": 0, "store": "no compaction", "posts_per_second": 21552.25, "p50_us": 2176, "p99_us": 17408, "max_us": 147456, "errors": 0}
+{"round": 0, "store": "no compaction, no growth", "posts_per_second": 24320.25, "p50_us": 1984, "p99_us": 12800, "max_us": 61440, "errors": 0}
+{"round": 0, "store": "tail grown as commits need it", "posts_per_second": 21392.25, "p50_us": 2176, "p99_us": 16384, "max_us": 69632, "errors": 0}
+{"round": 1, "store": "no compaction", "posts_per_second": 21942.75, "p50_us": 2176, "p99_us": 13312, "max_us": 65536, "errors": 0}
+{"round": 1, "store": "no compaction, no growth", "posts_per_second": 22688.25, "p50_us": 1984, "p99_us": 15360, "max_us": 163840, "errors": 0}
+{"round": 1, "store": "tail grown as commits need it", "posts_per_second": 21433.5, "p50_us": 2176, "p99_us": 14336, "max_us": 73728, "errors": 0}
+{"round": 1, "store": "as it is", "posts_per_second": 24960.25, "p50_us": 1984, "p99_us": 12800, "max_us": 73728, "errors": 0}
+{"round": 2, "store": "no compaction, no growth", "posts_per_second": 23088.0, "p50_us": 2176, "p99_us": 12800, "max_us": 53248, "errors": 0}
+{"round": 2, "store": "tail grown as commits need it", "posts_per_second": 22016.25, "p50_us": 2176, "p99_us": 15360, "max_us": 65536, "errors": 0}
+{"round": 2, "store": "as it is", "posts_per_second": 23760.25, "p50_us": 1984, "p99_us": 12288, "max_us": 90112, "errors": 0}
+{"round": 2, "store": "no compaction", "posts_per_second": 21280.25, "p50_us": 2176, "p99_us": 14336, "max_us": 59392, "errors": 0}
+{"round": 3, "store": "tail grown as commits need it", "posts_per_second": 22081.25, "p50_us": 2176, "p99_us": 13824, "max_us": 65536, "errors": 0}
+{"round": 3, "store": "as it is", "posts_per_second": 22594.75, "p50_us": 2176, "p99_us": 12288, "max_us": 81920, "errors": 0}
+{"round": 3, "store": "no compaction", "posts_per_second": 21104.25, "p50_us": 2176, "p99_us": 21504, "max_us": 139264, "errors": 0}
+{"round": 3, "store": "no compaction, no growth", "posts_per_second": 23419.25, "p50_us": 2176, "p99_us": 13312, "max_us": 57344, "errors": 0}
+{"round": 4, "store": "as it is", "posts_per_second": 22673.25, "p50_us": 2048, "p99_us": 15360, "max_us": 81920, "errors": 0}
+{"round": 4, "store": "no compaction", "posts_per_second": 23472.25, "p50_us": 2048, "p99_us": 14336, "max_us": 69632, "errors": 0}
+{"round": 4, "store": "no compaction, no growth", "posts_per_second": 23881.5, "p50_us": 1984, "p99_us": 16384, "max_us": 61440, "errors": 0}
+{"round": 4, "store": "tail grown as commits need it", "posts_per_second": 17632.25, "p50_us": 2304, "p99_us": 12800, "max_us": 589824, "errors": 0}
+{"round": 5, "store": "no compaction", "posts_per_second": 19040.25, "p50_us": 2176, "p99_us": 14336, "max_us": 655360, "errors": 0}
+{"round": 5, "store": "no compaction, no growth", "posts_per_second": 23552.25, "p50_us": 2048, "p99_us": 18432, "max_us": 59392, "errors": 0}
+{"round": 5, "store": "tail grown as commits need it", "posts_per_second": 22192.0, "p50_us": 2176, "p99_us": 15872, "max_us": 73728, "errors": 0}
+{"round": 5, "store": "as it is", "posts_per_second": 20928.25, "p50_us": 2176, "p99_us": 21504, "max_us": 77824, "errors": 0}
+{"round": 6, "store": "no compaction, no growth", "posts_per_second": 24608.25, "p50_us": 1984, "p99_us": 12800, "max_us": 61440, "errors": 0}
+{"round": 6, "store": "tail grown as commits need it", "posts_per_second": 19088.25, "p50_us": 2176, "p99_us": 15872, "max_us": 589824, "errors": 0}
+{"round": 6, "store": "as it is", "posts_per_second": 22576.25, "p50_us": 2048, "p99_us": 20480, "max_us": 77824, "errors": 0}
+{"round": 6, "store": "no compaction", "posts_per_second": 23184.25, "p50_us": 1984, "p99_us": 16384, "max_us": 73728, "errors": 0}
+{"round": 7, "store": "tail grown as commits need it", "posts_per_second": 22080.25, "p50_us": 2176, "p99_us": 14336, "max_us": 69632, "errors": 0}
+{"round": 7, "store": "as it is", "posts_per_second": 23088.25, "p50_us": 2048, "p99_us": 21504, "max_us": 65536, "errors": 0}
+{"round": 7, "store": "no compaction", "posts_per_second": 23051.75, "p50_us": 2048, "p99_us": 14336, "max_us": 65536, "errors": 0}
+{"round": 7, "store": "no compaction, no growth", "posts_per_second": 22586.5, "p50_us": 2048, "p99_us": 19456, "max_us": 57344, "errors": 0}
+{"store": "as it is", "rounds": 8, "posts_per_second_median": 22880.75, "p99_us_median": 17920.0, "max_us": 90112}
+{"store": "no compaction", "rounds": 8, "posts_per_second_median": 21747.5, "p99_us_median": 14336.0, "max_us": 655360}
+{"store": "no compaction, no growth", "rounds": 8, "posts_per_second_median": 23485.75, "p99_us_median": 14336.0, "max_us": 163840}
+{"store": "tail grown as commits need it", "rounds": 8, "posts_per_second_median": 21724.875, "p99_us_median": 14848.0, "max_us": 589824}
diff --git a/research/experiments/013-parallel-syncs/journal.py b/research/experiments/013-parallel-syncs/journal.py
new file mode 100644
index 0000000..5f49095
--- /dev/null
+++ b/research/experiments/013-parallel-syncs/journal.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""The tree store's commits under each store variant of variants.py, without
+HTTP: latency percentiles of research/experiments/010-one-sync-commit
+/commits.c, and how many ext4 journal commits (jbd2 transactions, read from
+/proc/fs/jbd2) each run caused. Rounds rotate the variants' order."""
+import argparse
+import json
+import re
+import shutil
+import statistics
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import variants
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[2]
+COMMITS = HERE.parent / '010-one-sync-commit' / 'commits.c'
+
+
+def transactions(device):
+    return int(re.match(r'(\d+) transactions', (Path('/proc/fs/jbd2') / device / 'info').read_text()).group(1))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--directory', type=Path, default=Path('/var/tmp'), help='on the device to measure')
+    parser.add_argument('--journal', default='mmcblk0p2-8', help='its /proc/fs/jbd2 entry')
+    parser.add_argument('--rounds', type=int, default=4)
+    parser.add_argument('--commits', type=int, default=4000)
+    args = parser.parse_args()
+    with tempfile.TemporaryDirectory() as work:
+        work = Path(work)
+        binaries = {}
+        for number, (name, patches) in enumerate(variants.PATCHES.items()):
+            source = (ROOT / 'store' / 'store-tree.c').read_text()
+            for old, new in patches:
+                assert old in source, old
+                source = source.replace(old, new)
+            (work / f'store-{number}.c').write_text(source)
+            binaries[name] = work / f'commits-{number}'
+            subprocess.run(['cc', '-O2', '-std=c17', f'-I{ROOT / "core"}', '-o', str(binaries[name]), str(COMMITS),
+                            str(work / f'store-{number}.c'), '-lpthread'], check=True)
+        names, runs = list(binaries), {name: [] for name in binaries}
+        for round_ in range(args.rounds):
+            for name in names[round_ % len(names):] + names[:round_ % len(names)]:
+                store = Path(tempfile.mkdtemp(dir=args.directory, prefix='pure-journal-'))
+                shutil.rmtree(store)
+                try:
+                    before = transactions(args.journal)
+                    out = subprocess.run([str(binaries[name]), str(store), str(args.commits)], capture_output=True,
+                                         text=True, check=True)
+                    measured = {**json.loads(out.stdout), 'journal_commits': transactions(args.journal) - before}
+                finally:
+                    shutil.rmtree(store, ignore_errors=True)
+                runs[name].append(measured)
+                print(json.dumps({'round': round_, 'store': name, **measured}), flush=True)
+        for name in names:
+            print(json.dumps({'store': name, 'commits': args.commits, **{key: statistics.median(r[key] for r in runs[name])
+                              for key in ('commits_per_second', 'p50_ms', 'p90_ms', 'p99_ms', 'journal_commits')}}))
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/research/experiments/013-parallel-syncs/variants.py b/research/experiments/013-parallel-syncs/variants.py
index 2c48ab6..ee49d2a 100644
--- a/research/experiments/013-parallel-syncs/variants.py
+++ b/research/experiments/013-parallel-syncs/variants.py
@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 """Where the tree store's extra sync work goes (docs/research/084-parallel-syncs.md).
 
-Builds pure-serve with three stores from store/store-tree.c: as it is; with no
-compaction; and with no compaction and a 64 MiB tail written at creation (no
-tail growth during a run). Then durable score posts (64 connections), the
+Builds pure-serve with stores from store/store-tree.c: as it is; with no
+compaction; with no compaction and a 64 MiB tail written at creation (no tail
+growth during a run); and as before 085, compacted files getting only 256 KiB
+of zeros and commits growing the tail as they need it. Then durable score posts (64 connections), the
 builds in rotated order per round, each on a fresh store after 64 warm-up
 posts. Prints one JSON line per run and a summary.
 """
@@ -25,6 +26,9 @@ PATCHES = {
     'no compaction': [('enum { COMPACT_SLACK = 1048576 };', 'enum { COMPACT_SLACK = 1 << 30 };')],
     'no compaction, no growth': [('enum { COMPACT_SLACK = 1048576 };', 'enum { COMPACT_SLACK = 1 << 30 };'),
                                  ('TAIL_STEP = 262144', 'TAIL_STEP = 67108864')],
+    # Before 085: a compacted file got 256 KiB of zeros, and commits grew the tail as they needed it.
+    'tail grown as commits need it': [('zero_fill(fd, head->end, cycle_end(head, replaced));',
+                                       'zero_fill(fd, head->end, tail_end(head->end));')],
 }