From 57ca99e5f4cdce7923e55347ed505e3ac43c3055 Mon Sep 17 00:00:00 2001 From: eunhhu Date: Sun, 4 Oct 2026 15:06:22 +0100 Subject: Say what views that compute cost the writes beside them (018 stage 3, research 119) --- .../034-view-bodies/2026-10-04-latency.jsonl | 25 +++ .../034-view-bodies/2026-10-04-threads-1.jsonl | 4 + .../034-view-bodies/2026-10-04-threads-2.jsonl | 4 + .../034-view-bodies/2026-10-04-views-first.jsonl | 19 +++ research/experiments/034-view-bodies/graph.aup | 79 ++++++++++ research/experiments/034-view-bodies/views.py | 168 +++++++++++++++++++++ 6 files changed, 299 insertions(+) create mode 100644 research/experiments/034-view-bodies/2026-10-04-latency.jsonl create mode 100644 research/experiments/034-view-bodies/2026-10-04-threads-1.jsonl create mode 100644 research/experiments/034-view-bodies/2026-10-04-threads-2.jsonl create mode 100644 research/experiments/034-view-bodies/2026-10-04-views-first.jsonl create mode 100644 research/experiments/034-view-bodies/graph.aup create mode 100755 research/experiments/034-view-bodies/views.py (limited to 'research/experiments') diff --git a/research/experiments/034-view-bodies/2026-10-04-latency.jsonl b/research/experiments/034-view-bodies/2026-10-04-latency.jsonl new file mode 100644 index 0000000..d074e8f --- /dev/null +++ b/research/experiments/034-view-bodies/2026-10-04-latency.jsonl @@ -0,0 +1,25 @@ +{"round": 1, "way": "writes", "writes_per_second": 120607, "write_p50_us": 120, "write_p99_us": 272, "write_max_us": 7168, "views_per_second": 0.0} +{"round": 1, "way": "both", "writes_per_second": 43296, "write_p50_us": 144, "write_p99_us": 640, "write_max_us": 98304, "views_per_second": 43.0} +{"round": 1, "way": "apart", "writes_per_second": 88452, "write_p50_us": 144, "write_p99_us": 864, "write_max_us": 7680, "views_per_second": 21.8} +{"round": 1, "way": "events", "writes_per_second": 187, "write_p50_us": 81920, "write_p99_us": 98304, "write_max_us": 139264, "views_per_second": 24.6} +{"round": 2, "way": "events", "writes_per_second": 187, "write_p50_us": 81920, "write_p99_us": 94208, "write_max_us": 98304, "views_per_second": 24.6} +{"round": 2, "way": "apart", "writes_per_second": 88993, "write_p50_us": 144, "write_p99_us": 704, "write_max_us": 6144, "views_per_second": 21.6} +{"round": 2, "way": "both", "writes_per_second": 30321, "write_p50_us": 152, "write_p99_us": 1472, "write_max_us": 102400, "views_per_second": 43.1} +{"round": 2, "way": "writes", "writes_per_second": 125767, "write_p50_us": 116, "write_p99_us": 232, "write_max_us": 5632, "views_per_second": 0.0} +{"round": 3, "way": "writes", "writes_per_second": 123756, "write_p50_us": 116, "write_p99_us": 272, "write_max_us": 5120, "views_per_second": 0.0} +{"round": 3, "way": "both", "writes_per_second": 85461, "write_p50_us": 108, "write_p99_us": 368, "write_max_us": 188416, "views_per_second": 22.3} +{"round": 3, "way": "apart", "writes_per_second": 54757, "write_p50_us": 232, "write_p99_us": 1984, "write_max_us": 9216, "views_per_second": 39.6} +{"round": 3, "way": "events", "writes_per_second": 188, "write_p50_us": 81920, "write_p99_us": 94208, "write_max_us": 98304, "views_per_second": 24.5} +{"round": 4, "way": "events", "writes_per_second": 186, "write_p50_us": 81920, "write_p99_us": 86016, "write_max_us": 86016, "views_per_second": 24.7} +{"round": 4, "way": "apart", "writes_per_second": 90794, "write_p50_us": 144, "write_p99_us": 672, "write_max_us": 6400, "views_per_second": 21.9} +{"round": 4, "way": "both", "writes_per_second": 29831, "write_p50_us": 152, "write_p99_us": 1664, "write_max_us": 155648, "views_per_second": 43.0} +{"round": 4, "way": "writes", "writes_per_second": 123193, "write_p50_us": 116, "write_p99_us": 240, "write_max_us": 5632, "views_per_second": 0.0} +{"round": 5, "way": "writes", "writes_per_second": 123248, "write_p50_us": 116, "write_p99_us": 288, "write_max_us": 5888, "views_per_second": 0.0} +{"round": 5, "way": "both", "writes_per_second": 25320, "write_p50_us": 152, "write_p99_us": 2432, "write_max_us": 163840, "views_per_second": 43.1} +{"round": 5, "way": "apart", "writes_per_second": 81905, "write_p50_us": 160, "write_p99_us": 768, "write_max_us": 6912, "views_per_second": 21.8} +{"round": 5, "way": "events", "writes_per_second": 185, "write_p50_us": 81920, "write_p99_us": 90112, "write_max_us": 94208, "views_per_second": 24.7} +{"round": 6, "way": "events", "writes_per_second": 184, "write_p50_us": 81920, "write_p99_us": 94208, "write_max_us": 102400, "views_per_second": 24.5} +{"round": 6, "way": "apart", "writes_per_second": 55420, "write_p50_us": 232, "write_p99_us": 1664, "write_max_us": 8192, "views_per_second": 40.9} +{"round": 6, "way": "both", "writes_per_second": 47134, "write_p50_us": 160, "write_p99_us": 992, "write_max_us": 118784, "views_per_second": 42.6} +{"round": 6, "way": "writes", "writes_per_second": 125222, "write_p50_us": 116, "write_p99_us": 240, "write_max_us": 6144, "views_per_second": 0.0} +{"nodes": 20000, "edges": 100000, "threads": 3, "askers": 2, "medians": {"writes": 123502.0, "both": 36808.5, "apart": 85178.5, "events": 186.5}, "write_p99_medians_us": {"writes": 256.0, "both": 1232.0, "apart": 816.0, "events": 94208.0}, "writes_kept": {"both": 29.8, "apart": 69.0, "events": 0.2}, "p99_both_over_apart": 1.51} diff --git a/research/experiments/034-view-bodies/2026-10-04-threads-1.jsonl b/research/experiments/034-view-bodies/2026-10-04-threads-1.jsonl new file mode 100644 index 0000000..1a52ac0 --- /dev/null +++ b/research/experiments/034-view-bodies/2026-10-04-threads-1.jsonl @@ -0,0 +1,4 @@ +{"round": 1, "way": "many", "views_per_second": 23.6} +{"round": 2, "way": "many", "views_per_second": 23.6} +{"round": 3, "way": "many", "views_per_second": 23.7} +{"nodes": 20000, "edges": 100000, "threads": 1, "askers": 2, "medians": {"many": 23.6}, "write_p99_medians_us": {}} diff --git a/research/experiments/034-view-bodies/2026-10-04-threads-2.jsonl b/research/experiments/034-view-bodies/2026-10-04-threads-2.jsonl new file mode 100644 index 0000000..4034b85 --- /dev/null +++ b/research/experiments/034-view-bodies/2026-10-04-threads-2.jsonl @@ -0,0 +1,4 @@ +{"round": 1, "way": "many", "views_per_second": 46.8} +{"round": 2, "way": "many", "views_per_second": 23.7} +{"round": 3, "way": "many", "views_per_second": 23.7} +{"nodes": 20000, "edges": 100000, "threads": 2, "askers": 2, "medians": {"many": 23.7}, "write_p99_medians_us": {}} diff --git a/research/experiments/034-view-bodies/2026-10-04-views-first.jsonl b/research/experiments/034-view-bodies/2026-10-04-views-first.jsonl new file mode 100644 index 0000000..abfdd45 --- /dev/null +++ b/research/experiments/034-view-bodies/2026-10-04-views-first.jsonl @@ -0,0 +1,19 @@ +{"round": 1, "way": "writes", "writes_per_second": 117606, "views_per_second": 0.0} +{"round": 1, "way": "both", "writes_per_second": 52913, "views_per_second": 16.1} +{"round": 1, "way": "apart", "writes_per_second": 53391, "views_per_second": 27.8} +{"round": 1, "way": "events", "writes_per_second": 189, "views_per_second": 24.9} +{"round": 1, "way": "one", "views_per_second": 13.3} +{"round": 1, "way": "many", "views_per_second": 22.2} +{"round": 2, "way": "many", "views_per_second": 23.9} +{"round": 2, "way": "one", "views_per_second": 13.5} +{"round": 2, "way": "events", "writes_per_second": 193, "views_per_second": 24.8} +{"round": 2, "way": "apart", "writes_per_second": 54521, "views_per_second": 33.6} +{"round": 2, "way": "both", "writes_per_second": 81288, "views_per_second": 22.4} +{"round": 2, "way": "writes", "writes_per_second": 118522, "views_per_second": 0.0} +{"round": 3, "way": "writes", "writes_per_second": 119471, "views_per_second": 0.0} +{"round": 3, "way": "both", "writes_per_second": 27513, "views_per_second": 37.6} +{"round": 3, "way": "apart", "writes_per_second": 89647, "views_per_second": 21.6} +{"round": 3, "way": "events", "writes_per_second": 188, "views_per_second": 24.6} +{"round": 3, "way": "one", "views_per_second": 13.7} +{"round": 3, "way": "many", "views_per_second": 23.3} +{"nodes": 20000, "edges": 100000, "threads": 3, "askers": 2, "writes_median": 118522, "with_views_median": 52913, "writes_kept": 44.6, "writes_kept_apart": 46.0, "writes_kept_events": 0.2, "views_one": 13.5, "views_many": 23.3, "scaling": 1.73} diff --git a/research/experiments/034-view-bodies/graph.aup b/research/experiments/034-view-bodies/graph.aup new file mode 100644 index 0000000..41302d9 --- /dev/null +++ b/research/experiments/034-view-bodies/graph.aup @@ -0,0 +1,79 @@ +# The app of research 119 (docs/design/018 stage 3): a graph whose +# distances a view computes in its body, beside the scoreboard's write +# (benchmarks/http/load's score) that the owner applies. +record Edge { + link = 0 + to = 0 +} +record Seen { + node = 0 + depth = 0 +} +record Step { + node = 0 +} +state edges: list of Edge by link +state nodes = 0 +state points = 0 +input size = message { + nodes: u32 +} +input load = message { + rows: list of Edge +} +input score = message { + points: u32 + name: bytes +} +input route = message { + start: u32 + goal: u32 +} +on size { + nodes = size.nodes +} +on load { + for r in load.rows { + edges.add(Edge { ..r }) + } +} +on score { + points = points + score.points +} +# The same search as an event, on the owner (to compare). +on route { + let seen: list of Seen by node + let queue: list of Step + queue.append(Step { node: route.start }) + seen.add(Seen { node: route.start }) + for i in 0 .. nodes { + if i >= queue.count { break } + let q = queue.first + i + for e in edges[q.node * 1048576 .. q.node * 1048576 + 1048576] { + if !seen.has(e.to) { + seen.add(Seen { node: e.to, depth: seen[q.node].depth + 1 }) + queue.append(Step { node: e.to }) + } + } + } + hops = if seen.has(route.goal) { seen[route.goal].depth } else { -1 } +} +state hops = -1 +view total = points +view distance(start: u32, goal: u32) { + let seen: list of Seen by node + let queue: list of Step + queue.append(Step { node: start }) + seen.add(Seen { node: start }) + for i in 0 .. nodes { + if i >= queue.count { break } + let q = queue.first + i + for e in edges[q.node * 1048576 .. q.node * 1048576 + 1048576] { + if !seen.has(e.to) { + seen.add(Seen { node: e.to, depth: seen[q.node].depth + 1 }) + queue.append(Step { node: e.to }) + } + } + } + return if seen.has(goal) { seen[goal].depth } else { -1 } +} diff --git a/research/experiments/034-view-bodies/views.py b/research/experiments/034-view-bodies/views.py new file mode 100755 index 0000000..92bdc7c --- /dev/null +++ b/research/experiments/034-view-bodies/views.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Views that compute, beside writes (docs/design/018 stage 3, research +119). pure-serve (THREADS connection threads, store in RAM, cores 0-2) +holds graph.aup with a random graph of NODES nodes and EDGES edges (seed +119); a second one like it (apart) shares the cores. In turns, for SECONDS +each, ROUNDS rounds: + writes benchmarks/http/load posts points (16 connections, keep-alive, + core 3): the owner applies them + both the same, while ASKERS connections ask the distance view (its + body a breadth first search) without pause, random arguments + apart the same, the askers asking the second host: the cores shared, + not the host + events the same, the askers posting the search as events (route): on + the owner + one, many the askers alone, 1 and ASKERS of them +Every view answer is checked against a model, made before any run is +timed: the askers start from 64 nodes picked beforehand (a model made while +asking would time Python, not the host). + +views.py PURE_SERVE AUP NODES EDGES THREADS ASKERS SECONDS ROUNDS [WAYS] a JSON line per run, then medians +(WAYS: the ways to run, by commas; all by default) +""" +import http.client +import json +import random +import shutil +import statistics +import struct +import subprocess +import sys +import tempfile +import threading +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +LOAD = HERE.parents[2] / 'benchmarks' / 'http' / 'load' + + +def main(): + serve, aup = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve() + nodes, edges_wanted, threads, askers, seconds, rounds = (int(a) for a in sys.argv[3:9]) + chosen = sys.argv[9].split(',') if len(sys.argv) > 9 else ['writes', 'both', 'apart', 'events', 'one', 'many'] + rng = random.Random(119) + pairs = set() + while len(pairs) < edges_wanted: + a, b = rng.randrange(nodes), rng.randrange(nodes) + if a != b: + pairs.add((a, b)) + edges = sorted(pairs) + out = [[] for _ in range(nodes)] + for a, b in edges: + out[a].append(b) + model = {} + + def far(a, b): + if a not in model: + depth, frontier = {a: 0}, [a] + for v in frontier: + for t in out[v]: + if t not in depth: + depth[t] = depth[v] + 1 + frontier.append(t) + model[a] = depth + return model[a].get(b, -1) + + work = Path(tempfile.mkdtemp(prefix='pure-034-', dir='/dev/shm')) + try: + subprocess.run([str(aup), 'build', str(HERE / 'graph.aup'), '-o', str(work / 'graph.pmod')], check=True) + host = subprocess.Popen(['taskset', '-c', '0-2', str(serve), '--module', str(work / 'graph.pmod'), '--store', + str(work / 'store'), '--port', '0', '--threads', str(threads)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + port = json.loads(host.stdout.readline())['port'] + apart = subprocess.Popen(['taskset', '-c', '0-2', str(serve), '--module', str(work / 'graph.pmod'), '--store', + str(work / 'apart'), '--port', '0', '--threads', str(threads)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + apart_port = json.loads(apart.stdout.readline())['port'] + for target in (port, apart_port): + setup = http.client.HTTPConnection('127.0.0.1', target, timeout=600) + + def post(path, body=b''): + setup.request('POST', path, body, {'Content-Length': str(len(body))}) + answer = setup.getresponse() + assert answer.status == 200 and answer.read() is not None, path + + post(f'/inputs/size?nodes={nodes}') + for k in range(0, len(edges), 80000): + post('/inputs/load', b''.join(struct.pack('