diff options
Diffstat (limited to 'research/experiments/004-direct-document-path/run.py')
| -rw-r--r-- | research/experiments/004-direct-document-path/run.py | 160 |
1 files changed, 160 insertions, 0 deletions
diff --git a/research/experiments/004-direct-document-path/run.py b/research/experiments/004-direct-document-path/run.py new file mode 100644 index 0000000..4987fd7 --- /dev/null +++ b/research/experiments/004-direct-document-path/run.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Compare the same durable document job through Pure and direct store calls.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import tempfile + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[2] +CONNECTIONS = ROOT / 'prototypes/connections' +EDITOR_SHA256 = '8ca6c191b98bf53a592ce24f2b5cd040e41f05c4ccdda7a072d5d7f79870c06a' + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def run(command, timeout=180): + result = subprocess.run([str(item) for item in command], capture_output=True, + text=True, timeout=timeout) + if result.returncode: + raise AssertionError((result.args, result.returncode, + result.stdout[-2000:], result.stderr[-2000:])) + return result + + +def compile_all(directory, cc, body_bytes): + flags = [cc, '-O2', '-std=c17', '-Wall', '-Wextra', '-Wpedantic', '-Werror', + f'-DPURE_BENCH_BODY_BYTES={body_bytes}'] + compiler = directory / 'pure-compile' + run([*flags, CONNECTIONS / 'compile-program.c', CONNECTIONS / 'program.c', + '-o', compiler]) + header = directory / 'embedded-plan.h' + run([compiler, CONNECTIONS / 'examples/editor.pure', header]) + common = [HERE / 'document-path.c', CONNECTIONS / 'store.c'] + direct = directory / 'direct' + pure = directory / 'pure' + run([*flags, '-DDIRECT_PATH', '-I' + str(CONNECTIONS), *common, '-o', direct]) + run([*flags, '-I' + str(CONNECTIONS), '-I' + str(directory), *common, + CONNECTIONS / 'store-port.c', CONNECTIONS / 'runtime.c', + CONNECTIONS / 'evaluate.c', '-o', pure]) + launcher = directory / 'peak-launcher' + run([*flags, HERE / 'peak-launcher.c', '-o', launcher]) + return {'direct': direct, 'pure': pure}, launcher, header + + +def sections(binary): + fields = run(['size', binary]).stdout.splitlines()[1].split() + return {'text': int(fields[0]), 'data': int(fields[1]), 'bss': int(fields[2])} + + +def trial(binary, launcher, store, documents): + result = run([launcher, binary, store, documents]) + measured = json.loads(result.stdout) + resources = dict(line.split('=', 1) for line in result.stderr.splitlines()) + measured['peak_rss_kib'] = int(resources['PURE_PEAK_RSS_KIB']) + measured['user_ns'] = int(resources['PURE_USER_NS']) + measured['system_ns'] = int(resources['PURE_SYSTEM_NS']) + objects = sorted(store.glob('obj-*')) + assert len(objects) == documents, len(objects) + measured['stored_file_bytes'] = sum(path.stat().st_size for path in objects) + final_files = hashlib.sha256() + for path in objects: + content = path.read_bytes() + final_files.update(path.name.encode('ascii')) + final_files.update(len(content).to_bytes(8, 'little')) + final_files.update(content) + measured['stored_files_sha256'] = final_files.hexdigest() + assert measured['documents'] == documents and measured['operations'] == 4 * documents + return measured + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--cc', default=os.environ.get('CC', 'cc')) + parser.add_argument('--documents', type=int, default=256) + parser.add_argument('--body-bytes', type=int, default=8192) + parser.add_argument('--pairs', type=int, default=4) + parser.add_argument('--work-root', type=Path, default=HERE / 'results', + help='filesystem for fresh stores; default is the repository filesystem') + parser.add_argument('--output', type=Path) + args = parser.parse_args() + if (not 1 <= args.documents <= 4096 or not 1 <= args.pairs <= 16 or + not 1 <= args.body_bytes <= 1024 * 1024): + parser.error('documents must be 1..4096, pairs 1..16 and body-bytes 1..1048576') + if args.output and args.output.exists(): + parser.error('evidence file already exists') + if platform.system() != 'Linux': + parser.error('this release RSS apparatus uses Linux wait4 units') + definition = CONNECTIONS / 'examples/editor.pure' + assert sha(definition) == EDITOR_SHA256, 'direct baseline needs review after editor.pure changes' + args.work_root.mkdir(parents=True, exist_ok=True) + filesystem = run(['df', '-T', args.work_root]).stdout.splitlines()[1].split() + with tempfile.TemporaryDirectory(prefix='pure-direct-document-', dir=args.work_root) as name: + root = Path(name) + binaries, launcher, prepared = compile_all(root, args.cc, args.body_bytes) + sizes = {name: sections(binary) for name, binary in binaries.items()} + binaries_sha = {name: sha(binary) for name, binary in binaries.items()} + # One small untimed run per path loads shared code and catches a broken + # comparison before the measured alternating, fresh-store trials. + warmups = {name: trial(binary, launcher, root / ('warm-' + name), 4) + for name, binary in binaries.items()} + assert warmups['direct']['digest'] == warmups['pure']['digest'] + samples = [] + for pair in range(args.pairs): + order = ('direct', 'pure') if pair % 2 == 0 else ('pure', 'direct') + entry = {'order': list(order)} + for variant in order: + entry[variant] = trial(binaries[variant], launcher, + root / f'{pair}-{variant}', args.documents) + assert entry['direct']['digest'] == entry['pure']['digest'] + assert entry['direct']['stored_file_bytes'] == entry['pure']['stored_file_bytes'] + assert entry['direct']['stored_files_sha256'] == entry['pure']['stored_files_sha256'] + samples.append(entry) + prepared_sha = sha(prepared) + medians = {variant: {'elapsed_ms': statistics.median( + sample[variant]['elapsed_ns'] for sample in samples) / 1e6, + 'peak_rss_kib': statistics.median( + sample[variant]['peak_rss_kib'] for sample in samples), + 'user_ms': statistics.median( + sample[variant]['user_ns'] for sample in samples) / 1e6, + 'system_ms': statistics.median( + sample[variant]['system_ns'] for sample in samples) / 1e6, + 'stages_ms': [statistics.median( + sample[variant]['stage_ns'][i] for sample in samples) / 1e6 + for i in range(4)]} + for variant in ('direct', 'pure')} + result = {'profile': 'same-durable-document-app-direct-versus-pure', + 'definition_sha256': sha(definition), 'prepared_sha256': prepared_sha, + 'documents_per_variant': args.documents, + 'operations_per_variant': args.documents * 4, + 'body_bytes': args.body_bytes, 'pairs': args.pairs, + 'filesystem': {'device': filesystem[0], 'type': filesystem[1], + 'mount': filesystem[-1]}, + 'sections': sizes, 'binary_sha256': binaries_sha, + 'samples': samples, 'medians': medians, + 'median_elapsed_ratio_pure_over_direct': + medians['pure']['elapsed_ms'] / medians['direct']['elapsed_ms'], + 'sources_sha256': {str(path.relative_to(ROOT)): sha(path) + for path in [HERE / 'document-path.c', HERE / 'peak-launcher.c', + HERE / 'run.py', CONNECTIONS / 'store.c', + CONNECTIONS / 'runtime.c', CONNECTIONS / 'evaluate.c', + CONNECTIONS / 'store-port.c', definition]}, + 'environment': {'platform': platform.platform(), 'machine': platform.machine(), + 'compiler': run([args.cc, '--version']).stdout.splitlines()[0], + 'size': run(['size', '--version']).stdout.splitlines()[0]}} + body = json.dumps(result, indent=2) + '\n' + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(body) + print(body, end='') + + +if __name__ == '__main__': + main() |