Count instructions where timing cannot tell 3% apart (110)

016's second stage passed its criterion: six interleaved pairs against
ee21aec gave TPC-C (RAM, W=1) -0.6% at the median and the bank's
queries 0.0% to +2.2%, no run interfered. The ranks query was slower
all six times, and chasing it showed that on this Pi timing cannot
separate 1-3%: two servers asked in turns disagree with themselves
(ee21aec to e02bfc9 +1.6%, e02bfc9 to 481642f -1.2%, ee21aec to 481642f
+2.8%), and a query whose instructions grew 0.6% ran 3.5% faster.

Instructions do not move so: pcount.c counts the server's user-mode
instructions with perf_event_open (no perf here), and psample.c samples
where they go. ranks had grown 0.4% in instructions (the rest was where
code and data landed), per_branch 3.6% and tables 20 instructions a row,
which 07047c3 halved. The six pairs of 07047c3 were interfered (another
session's build, load 4-12) and are kept apart, unused.

1 files changed, 75 insertions, 0 deletions

diff --git a/research/experiments/025-i64-cost/ab.py b/research/experiments/025-i64-cost/ab.py
new file mode 100755
index 0000000..a07ef72
--- /dev/null
+++ b/research/experiments/025-i64-cost/ab.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""The bank's four queries on two pure-serves at once, the same rows loaded
+into each, asked in turns (the order flipped every round): ab.py BEFORE AFTER
+ROUNDS ASKS. Prints each query's median of the rounds' medians, and in how
+many rounds AFTER was slower. One process a side: what differs between two
+processes (where code and data land) stays in the comparison, so run it
+again on fresh processes (ab-sessions.sh) before believing a percent."""
+import http.client, json, random, select, shutil, statistics, subprocess, sys, tempfile, time
+from pathlib import Path
+ROOT = Path(__file__).resolve().parents[3]
+sys.path.insert(0, str(ROOT / 'benchmarks' / 'bank'))
+import queries as q
+aup = str(ROOT.parent / 'aup' / 'build' / 'aup')
+serves = {'before': sys.argv[1], 'after': sys.argv[2]}
+rounds, asks = int(sys.argv[3]), int(sys.argv[4])
+work = Path(tempfile.mkdtemp(prefix='pure-ab-', dir='/dev/shm'))
+processes = {}
+try:
+    module = work / 'bank.pmod'
+    subprocess.run([aup, 'build', str(q.HERE / 'bank.aup'), '-o', str(module)], check=True)
+    compiled = {}
+    for name, (view, _) in q.QUERIES.items():
+        (work / f'{name}.aup').write_text(view)
+        subprocess.run([aup, 'query', str(q.HERE / 'bank.aup'), str(work / f'{name}.aup'), '-o', str(work / f'{name}.pmod')], check=True)
+        compiled[name] = (work / f'{name}.pmod').read_bytes()
+    ports = {}
+    for version, serve in serves.items():
+        p = subprocess.Popen(['taskset', '-c', '0-2', serve, '--module', str(module), '--store', str(work / f'store-{version}'),
+                              '--port', '0', '--timeout-ms', '600000'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        processes[version] = p
+        select.select([p.stdout], [], [], 10)
+        ports[version] = json.loads(p.stdout.readline())['port']
+    accounts, branches, transfers_n = 100000, 10, 20000
+    rng = random.Random(9)
+    tellers = 10 * branches
+    transfers = [(rng.randint(1, accounts), rng.randint(1, tellers), rng.randint(1, branches), rng.randint(-5000, 5000))
+                 for _ in range(transfers_n)]
+    for version, port in ports.items():
+        q.post_all(port, '/inputs/branch', [json.dumps({'bid': b}).encode() for b in range(1, branches + 1)], 1)
+        q.post_all(port, '/inputs/teller', [json.dumps({'tid': t, 'bid': (t - 1) // 10 + 1}).encode() for t in range(1, tellers + 1)], 1)
+        q.post_all(port, '/inputs/account', [json.dumps({'aid': a, 'bid': (a - 1) % branches + 1}).encode()
+                                             for a in range(1, accounts + 1)])
+        q.post_all(port, '/inputs/transfer', [json.dumps({'aid': a, 'tid': t, 'bid': b, 'delta': d}).encode()
+                                              for a, t, b, d in transfers], 1)
+    clients = {v: http.client.HTTPConnection('127.0.0.1', port, timeout=120) for v, port in ports.items()}
+    for v, client in clients.items():
+        for name, data in compiled.items():
+            client.request('PUT', f'/queries/{name}', data)
+            r = client.getresponse(); r.read(); assert r.status == 200
+    answers = {}
+    medians = {v: {name: [] for name in q.QUERIES} for v in serves}
+    for r in range(rounds):
+        order = ['before', 'after'] if r % 2 == 0 else ['after', 'before']
+        for name in q.QUERIES:
+            for v in order:
+                times = []
+                for _ in range(asks):
+                    began = time.perf_counter()
+                    clients[v].request('GET', f'/queries/{name}')
+                    res = clients[v].getresponse(); data = res.read()
+                    times.append(time.perf_counter() - began)
+                    assert res.status == 200
+                answers.setdefault(name, data)
+                assert answers[name] == data, name  # both answer alike
+                medians[v][name].append(statistics.median(times) * 1e3)
+    for name in q.QUERIES:
+        b, a = medians['before'][name], medians['after'][name]
+        wins = sum(1 for x, y in zip(b, a) if y > x)
+        print(json.dumps({'query': name, 'before_ms': round(statistics.median(b), 3), 'after_ms': round(statistics.median(a), 3),
+                          'change_pct': round((statistics.median(a) / statistics.median(b) - 1) * 100, 2),
+                          'rounds_after_slower': wins, 'rounds': len(b)}), flush=True)
+finally:
+    for p in processes.values():
+        p.terminate(); p.communicate(timeout=60)
+    shutil.rmtree(work)