diff options
Diffstat (limited to 'research/experiments')
| -rw-r--r-- | research/experiments/031-route-tokens/2026-10-04-tokens.jsonl | 33 | ||||
| -rw-r--r-- | research/experiments/031-route-tokens/tokens.py | 76 |
2 files changed, 109 insertions, 0 deletions
diff --git a/research/experiments/031-route-tokens/2026-10-04-tokens.jsonl b/research/experiments/031-route-tokens/2026-10-04-tokens.jsonl new file mode 100644 index 0000000..68fba88 --- /dev/null +++ b/research/experiments/031-route-tokens/2026-10-04-tokens.jsonl @@ -0,0 +1,33 @@ +{"mode": "keep-alive", "round": 1, "way": "open", "requests_per_second": 152428, "completed": 1524282, "2xx": 1524282, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 368, "p99_us": 576, "max_us": 21504} +{"mode": "keep-alive", "round": 1, "way": "bearing", "requests_per_second": 150441, "completed": 1504408, "2xx": 1504408, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 512, "max_us": 3712} +{"mode": "keep-alive", "round": 1, "way": "held", "requests_per_second": 150050, "completed": 1500509, "2xx": 1500509, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 480, "max_us": 1792} +{"mode": "keep-alive", "round": 2, "way": "held", "requests_per_second": 147988, "completed": 1479885, "2xx": 1479885, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 608, "max_us": 5376} +{"mode": "keep-alive", "round": 2, "way": "bearing", "requests_per_second": 151377, "completed": 1513777, "2xx": 1513777, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 480, "max_us": 1792} +{"mode": "keep-alive", "round": 2, "way": "open", "requests_per_second": 153512, "completed": 1535126, "2xx": 1535126, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 368, "p99_us": 576, "max_us": 2176} +{"mode": "keep-alive", "round": 3, "way": "open", "requests_per_second": 154031, "completed": 1540330, "2xx": 1540330, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 368, "p99_us": 576, "max_us": 1728} +{"mode": "keep-alive", "round": 3, "way": "bearing", "requests_per_second": 150391, "completed": 1503917, "2xx": 1503917, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 496, "max_us": 1792} +{"mode": "keep-alive", "round": 3, "way": "held", "requests_per_second": 149007, "completed": 1490066, "2xx": 1490066, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 576, "max_us": 2304} +{"mode": "keep-alive", "round": 4, "way": "held", "requests_per_second": 149336, "completed": 1493362, "2xx": 1493362, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 544, "max_us": 3456} +{"mode": "keep-alive", "round": 4, "way": "bearing", "requests_per_second": 150868, "completed": 1508682, "2xx": 1508682, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 480, "max_us": 2176} +{"mode": "keep-alive", "round": 4, "way": "open", "requests_per_second": 153821, "completed": 1538220, "2xx": 1538220, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 368, "p99_us": 576, "max_us": 1728} +{"mode": "keep-alive", "round": 5, "way": "open", "requests_per_second": 157154, "completed": 1571538, "2xx": 1571538, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 400, "p99_us": 544, "max_us": 1856} +{"mode": "keep-alive", "round": 5, "way": "bearing", "requests_per_second": 149931, "completed": 1499311, "2xx": 1499311, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 512, "max_us": 3456} +{"mode": "keep-alive", "round": 5, "way": "held", "requests_per_second": 150122, "completed": 1501220, "2xx": 1501220, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 416, "p99_us": 496, "max_us": 1792} +{"mode": "close", "round": 1, "way": "open", "requests_per_second": 22893, "completed": 228933, "2xx": 228933, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 896, "max_us": 6656} +{"mode": "close", "round": 1, "way": "bearing", "requests_per_second": 22754, "completed": 227538, "2xx": 227538, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 896, "max_us": 4864} +{"mode": "close", "round": 1, "way": "held", "requests_per_second": 22567, "completed": 225673, "2xx": 225673, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 928, "max_us": 7680} +{"mode": "close", "round": 2, "way": "held", "requests_per_second": 22338, "completed": 223380, "2xx": 223380, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 992, "max_us": 4864} +{"mode": "close", "round": 2, "way": "bearing", "requests_per_second": 22806, "completed": 228062, "2xx": 228062, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 928, "max_us": 5376} +{"mode": "close", "round": 2, "way": "open", "requests_per_second": 22868, "completed": 228679, "2xx": 228679, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 896, "max_us": 1344} +{"mode": "close", "round": 3, "way": "open", "requests_per_second": 22714, "completed": 227151, "2xx": 227151, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 928, "max_us": 5376} +{"mode": "close", "round": 3, "way": "bearing", "requests_per_second": 22713, "completed": 227134, "2xx": 227134, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 928, "max_us": 3456} +{"mode": "close", "round": 3, "way": "held", "requests_per_second": 22812, "completed": 228123, "2xx": 228123, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 896, "max_us": 5120} +{"mode": "close", "round": 4, "way": "held", "requests_per_second": 22660, "completed": 226607, "2xx": 226607, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 928, "max_us": 4864} +{"mode": "close", "round": 4, "way": "bearing", "requests_per_second": 22707, "completed": 227074, "2xx": 227074, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 928, "max_us": 4864} +{"mode": "close", "round": 4, "way": "open", "requests_per_second": 22912, "completed": 229117, "2xx": 229117, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 896, "max_us": 3456} +{"mode": "close", "round": 5, "way": "open", "requests_per_second": 22707, "completed": 227074, "2xx": 227074, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 928, "max_us": 5120} +{"mode": "close", "round": 5, "way": "bearing", "requests_per_second": 22705, "completed": 227051, "2xx": 227051, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 896, "max_us": 6400} +{"mode": "close", "round": 5, "way": "held", "requests_per_second": 22775, "completed": 227749, "2xx": 227749, "4xx": 0, "5xx": 0, "errors": 0, "p50_us": 736, "p99_us": 896, "max_us": 5376} +{"mode": "keep-alive", "open_median": 153821, "bearing_median": 150441, "held_median": 149336, "bearing_change": -2.2, "held_change": -2.92} +{"mode": "close", "open_median": 22868, "bearing_median": 22713, "held_median": 22660, "bearing_change": -0.68, "held_change": -0.91} +{"unauthorized": 0, "forbidden": 0} diff --git a/research/experiments/031-route-tokens/tokens.py b/research/experiments/031-route-tokens/tokens.py new file mode 100644 index 0000000..9b70be6 --- /dev/null +++ b/research/experiments/031-route-tokens/tokens.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""What an app's tokens cost its requests (docs/design/017's 3b, research 116). +A host of apps (one connection thread, two workers, stores in RAM, cores +0-2) holds the scoreboard (benchmarks/http/scores.aup) twice: open, with +no tokens, and held, whose tokens file grants a write token. The load +(benchmarks/http/load, core 3) reads its total view for SECONDS on +CONNECTIONS connections, keep-alive (a connection's token hashed once) or +close (a connection a request: hashed every time): open with no token, +open bearing the token all the same (its head as long, nothing checked), +and held; the three in turns, their order turned each round. + +tokens.py PURE_SERVE SECONDS ROUNDS a JSON line per run, then medians +""" +import hashlib +import json +import os +import shutil +import statistics +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[2] +AUP = ROOT.parent / 'aup' / 'build' / 'aup' +LOAD = ROOT / 'benchmarks' / 'http' / 'load' +TOKEN = 'a' * 64 + + +def load(port, way, connections, seconds, mode): + env = dict(os.environ) + app = 'held' if way == 'held' else 'open' + if way != 'open': + env['PURE_LOAD_TOKEN'] = TOKEN + out = subprocess.run(['taskset', '-c', '3', str(LOAD), str(port), str(connections), str(seconds), 'total', '1', mode, + '16', f'/{app}'], capture_output=True, text=True, env=env, check=True).stdout + return json.loads(out) + + +def main(): + serve, seconds, rounds = Path(sys.argv[1]).resolve(), int(sys.argv[2]), int(sys.argv[3]) + work = Path(tempfile.mkdtemp(prefix='pure-031-', dir='/dev/shm')) + try: + module = work / 'scores.pmod' + subprocess.run([str(AUP), 'build', str(ROOT / 'benchmarks' / 'http' / 'scores.aup'), '-o', str(module)], check=True) + (work / 'tokens').write_text(f'write {hashlib.sha256(TOKEN.encode()).hexdigest()}\n') + (work / 'apps.conf').write_text(f"open {module} {work / 'open'}\nheld {module} {work / 'held'} " + f"tokens={work / 'tokens'}\n") + host = subprocess.Popen(['taskset', '-c', '0-2', str(serve), '--apps', str(work / 'apps.conf'), '--port', '0', + '--threads', '1', '--workers', '2'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + port = json.loads(host.stdout.readline())['port'] + rates = {} + for mode, connections in (('keep-alive', 64), ('close', 16)): + for round_ in range(1, rounds + 1): + ways = ['open', 'bearing', 'held'] + for way in (ways if round_ % 2 else ways[::-1]): + result = load(port, way, connections, seconds, mode) + assert result['2xx'] == result['completed'] and not result['errors'], result + rates.setdefault((mode, way), []).append(result['requests_per_second']) + print(json.dumps({'mode': mode, 'round': round_, 'way': way, **result}), flush=True) + host.terminate() + _, error = host.communicate(timeout=60) + stats = json.loads(error.decode().strip().splitlines()[-1]) + for mode in ('keep-alive', 'close'): + open_, bearing, held = (statistics.median(rates[(mode, way)]) for way in ('open', 'bearing', 'held')) + print(json.dumps({'mode': mode, 'open_median': open_, 'bearing_median': bearing, 'held_median': held, + 'bearing_change': round((bearing / open_ - 1) * 100, 2), + 'held_change': round((held / open_ - 1) * 100, 2)}), flush=True) + print(json.dumps({'unauthorized': stats['unauthorized'], 'forbidden': stats['forbidden']}), flush=True) + finally: + shutil.rmtree(work) + + +if __name__ == '__main__': + main() |