diff options
Diffstat (limited to 'benchmarks')
| -rw-r--r-- | benchmarks/http/README.md | 10 | ||||
| -rw-r--r-- | benchmarks/http/run.py | 118 |
2 files changed, 91 insertions, 37 deletions
diff --git a/benchmarks/http/README.md b/benchmarks/http/README.md index 3c6eb4f..e26174b 100644 --- a/benchmarks/http/README.md +++ b/benchmarks/http/README.md @@ -10,3 +10,13 @@ make -C ../../prototypes/connections BUILD=build/tree STORE=store-tree.c build/t aup build ../../../aup/examples/editor.aup -o editor.pmod python3 run.py --module editor.pmod --directory /var/tmp # 저장소를 둘 장치에서 ``` + +저장 보장 쓰기는 장치 상태가 같을 때만 비교된다. SD 카드는 동기 쓰기가 이어지면 느려지므로 실행 순서만으로 +결과가 몇 배 달라진다. 쓰기를 비교할 때는 라운드를 여러 번 돌린다. 라운드마다 서버 순서를 회전하고, 서버마다 +새 저장소를 쓰며, 쓰기 작업 직전에 장치의 fdatasync 지연(제자리 4 KiB)을 재서 결과 옆에 적는다. 표는 라운드의 +중앙값이다. + +```sh +python3 run.py --module editor.pmod --workloads put --rounds 5 --seconds 3 \ + --only pure-serve,pure-serve-3,axum-durable,axum-atomic +``` diff --git a/benchmarks/http/run.py b/benchmarks/http/run.py index 8ce1474..f9fdbc0 100644 --- a/benchmarks/http/run.py +++ b/benchmarks/http/run.py @@ -4,7 +4,15 @@ same contract, on the same machine, with the same load generator. Servers are pinned to CPUs 0-2 and the load to CPU 3; stores live under --directory (use the device you care about: fsync costs are the device's). -Prints one JSON object per (server, workload) and a summary table. +Prints one JSON object per (server, workload, round) and a summary table of +medians over rounds. + +Durable writes are only comparable on a device in the same state. An SD card +slows down under sustained syncs, so the order of runs alone can change a +result several times over. Each round therefore runs the servers in a +rotated order, each on a fresh store, and every write workload first times +the device (an in-place 4 KiB write + fdatasync); the table shows that probe +next to each result. """ import argparse import json @@ -12,6 +20,7 @@ import os import select import shutil import signal +import statistics import struct import subprocess import sys @@ -51,6 +60,25 @@ def put(port, key, body, expected): assert response.status == 200, (key, response.status) +def probe_ms(directory, count=16): + """Median time of an in-place 4 KiB write + fdatasync on the store device.""" + path = directory / 'device-probe' + fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) + try: + os.pwrite(fd, bytes(count * 4096), 0) + os.fsync(fd) + times = [] + for i in range(count): + start = time.perf_counter() + os.pwrite(fd, b'p' * 4096, i * 4096) + os.fdatasync(fd) + times.append((time.perf_counter() - start) * 1e3) + finally: + os.close(fd) + path.unlink() + return round(statistics.median(times), 2) + + def load(port, connections, seconds, method, keys, mode, body=16): output = subprocess.run(['taskset', '-c', '3', str(HERE / 'load'), str(port), str(connections), str(seconds), method, str(keys), mode, str(body)], capture_output=True, text=True, check=True) @@ -63,6 +91,9 @@ def main(): parser.add_argument('--directory', type=Path, default=Path('/var/tmp')) parser.add_argument('--seconds', type=float, default=5) parser.add_argument('--only', default='') + parser.add_argument('--workloads', default='', help='run only workloads whose label starts with one of these') + parser.add_argument('--rounds', type=int, default=1, help='repeat every server, rotating their order') + parser.add_argument('--pause', type=float, default=0, help='seconds to rest the device between runs') parser.add_argument('--budget', type=int, default=64 << 20, help="Pure's payload budget in bytes") args = parser.parse_args() axum = HERE / 'axum-baseline/target/release/axum-baseline' @@ -85,50 +116,63 @@ def main(): ('put 16B, 64 conn, keep-alive', dict(connections=64, method='put', keys=64, mode='keep-alive')), ('put 16B, 8 conn, keep-alive', dict(connections=8, method='put', keys=8, mode='keep-alive')), ] + if args.workloads: + workloads = [w for w in workloads if w[0].startswith(tuple(args.workloads.split(',')))] + chosen = [name for name in servers if not args.only or name in args.only.split(',')] results = [] try: - for name, command in servers.items(): - if args.only and name not in args.only.split(','): - continue - directory = base / name - process, port = start(command(directory)) - idle = rss_kib(process.pid, 'VmRSS:') - for key in range(64): - put(port, f'k{key}', b'x' * 56, 0) - big = b'B' * (1 << 20) - put(port, 'big', big, 0) - for label, workload in workloads: - keys = workload['keys'] - if workload.get('big'): - # the 1 MiB document is k0 for this workload - put(port, 'k0', big, 1) - if workload['method'] == 'put': - # fresh keys per run so every connection starts at If-None-Match: * - pass - measured = load(port, workload['connections'], args.seconds, workload['method'], keys, - workload['mode']) - measured.update(server=name, workload=label, rss_kib=rss_kib(process.pid, 'VmRSS:'), - ok_per_second=round(measured['2xx'] / args.seconds)) - results.append(measured) - print(json.dumps(measured), flush=True) - peak = rss_kib(process.pid, 'VmHWM:') - process.send_signal(signal.SIGINT if name.startswith('axum') else signal.SIGTERM) - process.communicate(timeout=30) - for r in results: - if r['server'] == name: - r['idle_rss_kib'], r['peak_rss_kib'] = idle, peak + for round_ in range(args.rounds): + turn = round_ % len(chosen) + for name in chosen[turn:] + chosen[:turn]: + if args.pause: + time.sleep(args.pause) + directory = base / f'{name}-{round_}' + process, port = start(servers[name](directory)) + idle = rss_kib(process.pid, 'VmRSS:') + for key in range(64): + put(port, f'k{key}', b'x' * 56, 0) + for label, workload in workloads: + if workload.get('big'): + # the 1 MiB document is k0 for this workload + put(port, 'k0', b'B' * (1 << 20), 1) + probe = probe_ms(base) if workload['method'] == 'put' else None + measured = load(port, workload['connections'], args.seconds, workload['method'], + workload['keys'], workload['mode']) + measured.update(server=name, workload=label, round=round_, probe_ms=probe, + rss_kib=rss_kib(process.pid, 'VmRSS:'), + ok_per_second=round(measured['2xx'] / args.seconds), idle_rss_kib=idle) + results.append(measured) + print(json.dumps(measured), flush=True) + peak = rss_kib(process.pid, 'VmHWM:') + process.send_signal(signal.SIGINT if name.startswith('axum') else signal.SIGTERM) + process.communicate(timeout=30) + shutil.rmtree(directory, ignore_errors=True) + for r in results: + if r['server'] == name and r['round'] == round_: + r['peak_rss_kib'] = peak finally: shutil.rmtree(base, ignore_errors=True) + median = lambda rows, field: statistics.median(r[field] for r in rows) print() - print(f"{'workload':32s} {'server':14s} {'2xx/s':>9s} {'p50 us':>8s} {'p99 us':>8s} {'errors':>7s} {'non-2xx':>8s}") - for r in results: - print(f"{r['workload']:32s} {r['server']:14s} {r['ok_per_second']:9d} {r['p50_us']:8d} {r['p99_us']:8d} " - f"{r['errors']:7d} {r['4xx'] + r['5xx']:8d}") + print(f"medians over {args.rounds} round(s)") + print(f"{'workload':32s} {'server':14s} {'2xx/s':>9s} {'p50 us':>8s} {'p99 us':>8s} {'errors':>7s} " + f"{'non-2xx':>8s} {'probe ms':>9s}") + for label, _ in workloads: + for name in chosen: + rows = [r for r in results if r['server'] == name and r['workload'] == label] + if not rows: + continue + probes = [r['probe_ms'] for r in rows if r['probe_ms'] is not None] + probe = f"{min(probes):.1f}-{max(probes):.1f}" if probes else '-' + print(f"{label:32s} {name:14s} {median(rows, 'ok_per_second'):9.0f} {median(rows, 'p50_us'):8.0f} " + f"{median(rows, 'p99_us'):8.0f} {sum(r['errors'] for r in rows):7d} " + f"{sum(r['4xx'] + r['5xx'] for r in rows):8d} {probe:>9s}") print() - for name in servers: + for name in chosen: rows = [r for r in results if r['server'] == name] if rows: - print(f"{name:14s} idle RSS {rows[0]['idle_rss_kib']} KiB, peak RSS {rows[0]['peak_rss_kib']} KiB") + print(f"{name:14s} idle RSS {median(rows, 'idle_rss_kib'):.0f} KiB, " + f"peak RSS {max(r['peak_rss_kib'] for r in rows)} KiB") if __name__ == '__main__': |