Measure how long an app changing module cannot answer, against a host restart (030)

An app of a million items (a list grown by doubling) or of a thousand, in
a host of apps, its store in RAM: the longest gap between two answered
writes while it changes module on SIGHUP (stopped and started again from
its store, 71e319b), and while the host is restarted. A thousand items:
1.59 ms against 3.05 (median of 20 and 10). A million: 150.64 ms against
149.73 (10 and 5): both are the store read again (pure_app_start, 142
ms), so changing does not answer sooner than restarting.

6 files changed, 293 insertions, 0 deletions

diff --git a/research/experiments/030-app-lifecycle/2026-10-04-window-stop-start-1000.jsonl b/research/experiments/030-app-lifecycle/2026-10-04-window-stop-start-1000.jsonl
new file mode 100644
index 0000000..93e8e93
--- /dev/null
+++ b/research/experiments/030-app-lifecycle/2026-10-04-window-stop-start-1000.jsonl
@@ -0,0 +1,31 @@
+{"items": 1024, "way": "changed", "gap_ms": 0.7, "reading_ms": 0.69}
+{"items": 1024, "way": "changed", "gap_ms": 5.06, "reading_ms": 1.65}
+{"items": 1024, "way": "changed", "gap_ms": 1.65, "reading_ms": 0.63}
+{"items": 1024, "way": "changed", "gap_ms": 2.97, "reading_ms": 0.63}
+{"items": 1024, "way": "changed", "gap_ms": 0.49, "reading_ms": 0.53}
+{"items": 1024, "way": "changed", "gap_ms": 0.59, "reading_ms": 0.61}
+{"items": 1024, "way": "changed", "gap_ms": 5.67, "reading_ms": 1.59}
+{"items": 1024, "way": "changed", "gap_ms": 0.59, "reading_ms": 0.6}
+{"items": 1024, "way": "changed", "gap_ms": 1.68, "reading_ms": 1.67}
+{"items": 1024, "way": "changed", "gap_ms": 1.58, "reading_ms": 1.67}
+{"items": 1024, "way": "changed", "gap_ms": 2.04, "reading_ms": 0.62}
+{"items": 1024, "way": "changed", "gap_ms": 0.49, "reading_ms": 0.54}
+{"items": 1024, "way": "changed", "gap_ms": 1.46, "reading_ms": 1.63}
+{"items": 1024, "way": "changed", "gap_ms": 0.57, "reading_ms": 0.57}
+{"items": 1024, "way": "changed", "gap_ms": 1.59, "reading_ms": 1.7}
+{"items": 1024, "way": "changed", "gap_ms": 1.72, "reading_ms": 1.65}
+{"items": 1024, "way": "changed", "gap_ms": 1.85, "reading_ms": 0.66}
+{"items": 1024, "way": "changed", "gap_ms": 0.53, "reading_ms": 0.57}
+{"items": 1024, "way": "changed", "gap_ms": 1.58, "reading_ms": 1.66}
+{"items": 1024, "way": "changed", "gap_ms": 2.13, "reading_ms": 1.6}
+{"items": 1024, "way": "restarted", "gap_ms": 2.58}
+{"items": 1024, "way": "restarted", "gap_ms": 4.15}
+{"items": 1024, "way": "restarted", "gap_ms": 3.09}
+{"items": 1024, "way": "restarted", "gap_ms": 2.44}
+{"items": 1024, "way": "restarted", "gap_ms": 3.52}
+{"items": 1024, "way": "restarted", "gap_ms": 2.71}
+{"items": 1024, "way": "restarted", "gap_ms": 3.01}
+{"items": 1024, "way": "restarted", "gap_ms": 4.08}
+{"items": 1024, "way": "restarted", "gap_ms": 3.9}
+{"items": 1024, "way": "restarted", "gap_ms": 2.44}
+{"items": 1024, "changed_median_ms": 1.59, "restarted_median_ms": 3.05, "refused": 172, "answered": 62355}
diff --git a/research/experiments/030-app-lifecycle/2026-10-04-window-stop-start-1000000.jsonl b/research/experiments/030-app-lifecycle/2026-10-04-window-stop-start-1000000.jsonl
new file mode 100644
index 0000000..03e5ee2
--- /dev/null
+++ b/research/experiments/030-app-lifecycle/2026-10-04-window-stop-start-1000000.jsonl
@@ -0,0 +1,16 @@
+{"items": 1048576, "way": "changed", "gap_ms": 150.55, "reading_ms": 150.51}
+{"items": 1048576, "way": "changed", "gap_ms": 151.01, "reading_ms": 151.01}
+{"items": 1048576, "way": "changed", "gap_ms": 150.65, "reading_ms": 150.67}
+{"items": 1048576, "way": "changed", "gap_ms": 149.57, "reading_ms": 149.55}
+{"items": 1048576, "way": "changed", "gap_ms": 150.7, "reading_ms": 150.72}
+{"items": 1048576, "way": "changed", "gap_ms": 151.16, "reading_ms": 151.22}
+{"items": 1048576, "way": "changed", "gap_ms": 150.62, "reading_ms": 150.66}
+{"items": 1048576, "way": "changed", "gap_ms": 150.63, "reading_ms": 150.6}
+{"items": 1048576, "way": "changed", "gap_ms": 150.93, "reading_ms": 150.95}
+{"items": 1048576, "way": "changed", "gap_ms": 150.11, "reading_ms": 150.11}
+{"items": 1048576, "way": "restarted", "gap_ms": 149.9}
+{"items": 1048576, "way": "restarted", "gap_ms": 148.92}
+{"items": 1048576, "way": "restarted", "gap_ms": 149.04}
+{"items": 1048576, "way": "restarted", "gap_ms": 149.73}
+{"items": 1048576, "way": "restarted", "gap_ms": 150.4}
+{"items": 1048576, "changed_median_ms": 150.64, "restarted_median_ms": 149.73, "refused": 18720, "answered": 33810}
diff --git a/research/experiments/030-app-lifecycle/items-next.aup b/research/experiments/030-app-lifecycle/items-next.aup
new file mode 100644
index 0000000..7a1917a
--- /dev/null
+++ b/research/experiments/030-app-lifecycle/items-next.aup
@@ -0,0 +1,29 @@
+record Item {
+  n = 0
+}
+state items: list of Item
+state more: list of Item
+state hits = 0
+input seed = message { n: number }
+input grow = message { step: number }
+input add = message { step: number }
+on seed {
+  items.append(Item { n: seed.n })
+}
+on grow {
+  for i in items {
+    more.append(Item { n: i.n + grow.step })
+  }
+  for m in more {
+    items.append(Item { n: m.n })
+  }
+  for m in more {
+    more.remove(m)
+  }
+}
+on add {
+  hits = hits + add.step
+}
+view count = count(i in items where i.n > 0)
+view total = hits
+view size = count(i in items where i.n >= 0)
diff --git a/research/experiments/030-app-lifecycle/items.aup b/research/experiments/030-app-lifecycle/items.aup
new file mode 100644
index 0000000..b6d928b
--- /dev/null
+++ b/research/experiments/030-app-lifecycle/items.aup
@@ -0,0 +1,28 @@
+record Item {
+  n = 0
+}
+state items: list of Item
+state more: list of Item
+state hits = 0
+input seed = message { n: number }
+input grow = message { step: number }
+input add = message { step: number }
+on seed {
+  items.append(Item { n: seed.n })
+}
+on grow {
+  for i in items {
+    more.append(Item { n: i.n + grow.step })
+  }
+  for m in more {
+    items.append(Item { n: m.n })
+  }
+  for m in more {
+    more.remove(m)
+  }
+}
+on add {
+  hits = hits + add.step
+}
+view count = count(i in items where i.n > 0)
+view total = hits
diff --git a/research/experiments/030-app-lifecycle/runs.sh b/research/experiments/030-app-lifecycle/runs.sh
new file mode 100755
index 0000000..601f754
--- /dev/null
+++ b/research/experiments/030-app-lifecycle/runs.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+# The runs of research 115 (docs/design/017's stage 3), from the repository's
+# root, in steps, each under research/guard.py:
+#   runs.sh build NAME COMMIT   pure-serve of COMMIT, with its pure-crypto.so,
+#                               in ~/.cache/pure-030/NAME
+#   runs.sh window NAME ITEMS CHANGES RESTARTS [MODULE]   window.py on that
+#                               build, into DATE-window-NAME-ITEMS[-MODULE].jsonl
+set -e
+out=research/experiments/030-app-lifecycle
+builds=$HOME/.cache/pure-030
+day=$(date +%F)
+case "$1" in
+build)
+    rm -rf "$builds/src-$2"
+    git worktree add -f -q --detach "$builds/src-$2" "$3"
+    make -s -C "$builds/src-$2" build/pure-serve build/pure-crypto.so
+    mkdir -p "$builds/$2"
+    cp "$builds/src-$2/build/pure-serve" "$builds/src-$2/build/pure-crypto.so" "$builds/$2/"
+    git worktree remove --force "$builds/src-$2" ;;
+window)
+    until awk '{ exit !($1 < 1.5) }' /proc/loadavg; do sleep 10; done
+    python3 research/guard.py --most 2048 -- python3 $out/window.py "$builds/$2/pure-serve" "$3" "$4" "$5" ${6:+"$6"} \
+        > "$out/$day-window-$2-$3${6:+-$6}.jsonl" ;;
+*) echo "runs.sh build NAME COMMIT | window NAME ITEMS CHANGES RESTARTS [MODULE]" >&2; exit 2 ;;
+esac
diff --git a/research/experiments/030-app-lifecycle/window.py b/research/experiments/030-app-lifecycle/window.py
new file mode 100755
index 0000000..3ec21a5
--- /dev/null
+++ b/research/experiments/030-app-lifecycle/window.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""How long an app cannot answer while it changes module on a running host
+(docs/design/017's stage 3), against restarting the host (research 115).
+
+An app of items.aup holds ITEMS items (a list grown by doubling) in a host
+of apps, its store in RAM, the host on cores 0-2. A client on core 3 posts
+writes to it one after another on one connection; the time an app cannot
+answer is the longest gap between two answered writes (200) around the
+change. Changed: SIGHUP after its line names the other of two module files
+(items.pmod, items-next.pmod: the same states, a view more), so each
+reading starts it again on another module. Restarted: SIGTERM, then the
+host started again (the client reconnecting at once).
+
+window.py PURE_SERVE ITEMS CHANGES RESTARTS   a JSON line per change and per
+                                              restart, then one with medians
+"""
+import http.client
+import json
+import os
+import select
+import shutil
+import signal
+import socket
+import statistics
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+AUP = HERE.parents[3] / 'aup' / 'build' / 'aup'
+
+
+class Host:
+    def __init__(self, serve, conf, port):
+        self.process = subprocess.Popen(['taskset', '-c', '0-2', str(serve), '--apps', str(conf), '--port', str(port)],
+                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        ready, _, _ = select.select([self.process.stdout], [], [], 600)
+        assert ready, self.process.stderr.read()
+        self.info = json.loads(self.process.stdout.readline())
+
+    def reading(self):
+        self.process.send_signal(signal.SIGHUP)
+        ready, _, _ = select.select([self.process.stdout], [], [], 600)
+        assert ready
+        return json.loads(self.process.stdout.readline())
+
+    def stop(self):
+        self.process.terminate()
+        _, error = self.process.communicate(timeout=600)
+        assert self.process.returncode == 0, error[-500:]
+
+
+class Writer(threading.Thread):
+    """Writes one after another; the times of those answered 200."""
+
+    def __init__(self, port):
+        super().__init__(daemon=True)
+        self.port, self.answered, self.refused, self.stopping = port, [], 0, False
+
+    def run(self):
+        os.sched_setaffinity(0, {3})
+        connection = None
+        while not self.stopping:
+            try:
+                if connection is None:
+                    connection = http.client.HTTPConnection('127.0.0.1', self.port, timeout=600)
+                connection.request('POST', '/x/inputs/add?step=1', b'', {'Content-Length': '0'})
+                answer = connection.getresponse()
+                answer.read()
+                if answer.status == 200:
+                    self.answered.append(time.monotonic())
+                else:
+                    self.refused += 1
+            except (OSError, http.client.HTTPException):  # the host is down: again at once
+                connection = None
+                time.sleep(0.0002)
+
+    def gap(self, since):
+        """The longest gap between answered writes from since on (and the one before it)."""
+        times = [t for t in self.answered if t >= since]
+        before = [t for t in self.answered if t < since]
+        if before:
+            times = before[-1:] + times
+        return max(b - a for a, b in zip(times, times[1:]))
+
+
+def free_port():
+    with socket.socket() as probe:
+        probe.bind(('127.0.0.1', 0))
+        return probe.getsockname()[1]
+
+
+def grow(port, items):
+    connection = http.client.HTTPConnection('127.0.0.1', port, timeout=600)
+
+    def post(path):
+        connection.request('POST', path, b'', {'Content-Length': '0'})
+        answer = connection.getresponse()
+        answer.read()
+        assert answer.status == 200, (path, answer.status)
+
+    post('/x/inputs/seed?n=1')
+    count = 1
+    while count < items:
+        post('/x/inputs/grow?step=1')
+        count *= 2
+    connection.close()
+    return count
+
+
+def main():
+    serve, items, changes, restarts = Path(sys.argv[1]).resolve(), int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])
+    work = Path(tempfile.mkdtemp(prefix='pure-030-', dir='/dev/shm'))
+    try:
+        modules = {}
+        for name in ('items', 'items-next'):
+            modules[name] = work / f'{name}.pmod'
+            subprocess.run([str(AUP), 'build', str(HERE / f'{name}.aup'), '-o', str(modules[name])], check=True)
+        conf, store, port = work / 'apps.conf', work / 'store', free_port()
+
+        def write(module):
+            conf.write_text(f"x {modules[module]} {store}\n")
+
+        write('items')
+        host = Host(serve, conf, port)
+        count = grow(port, items)
+        writer = Writer(port)
+        writer.start()
+        time.sleep(0.5)
+        gaps = {'changed': [], 'restarted': []}
+        for n in range(changes):
+            write('items-next' if n % 2 == 0 else 'items')
+            since = time.monotonic()
+            began = time.perf_counter()
+            said = host.reading()
+            took = time.perf_counter() - began
+            assert said['apps'][0]['did'] == 'restarted' and said['apps'][0]['running'], said
+            time.sleep(0.3)
+            gaps['changed'].append(writer.gap(since))
+            print(json.dumps({'items': count, 'way': 'changed', 'gap_ms': round(gaps['changed'][-1] * 1000, 2),
+                              'reading_ms': round(took * 1000, 2)}), flush=True)
+        for n in range(restarts):
+            since = time.monotonic()
+            host.stop()
+            host = Host(serve, conf, port)
+            time.sleep(0.3)
+            gaps['restarted'].append(writer.gap(since))
+            print(json.dumps({'items': count, 'way': 'restarted', 'gap_ms': round(gaps['restarted'][-1] * 1000, 2)}),
+                  flush=True)
+        writer.stopping = True
+        writer.join()
+        host.stop()
+        print(json.dumps({'items': count, 'changed_median_ms': round(statistics.median(gaps['changed']) * 1000, 2),
+                          'restarted_median_ms': round(statistics.median(gaps['restarted']) * 1000, 2),
+                          'refused': writer.refused, 'answered': len(writer.answered)}), flush=True)
+    finally:
+        shutil.rmtree(work)
+
+
+if __name__ == '__main__':
+    main()