Establish native micro runtime checkpoint

1 files changed, 361 insertions, 0 deletions

diff --git a/prototypes/connections/package_native.py b/prototypes/connections/package_native.py
new file mode 100644
index 0000000..7ee9c4f
--- /dev/null
+++ b/prototypes/connections/package_native.py
@@ -0,0 +1,361 @@
+#!/usr/bin/env python3
+"""Bundle the selected Linux document app without folding hosts into the core."""
+import argparse
+import gzip
+import hashlib
+import io
+import json
+from pathlib import Path
+import platform
+import re
+import subprocess
+import tarfile
+import tempfile
+
+HERE = Path(__file__).resolve().parent
+NEEDED = re.compile(r'\(NEEDED\).*Shared library: \[([^]]+)\]')
+ROOT_NAME = 'pure-document'
+PAGED_STORAGE = """# Selected page storage format
+
+This preview uses the PUREP001 16 KiB-page store. It cannot open PUREB001
+flat-store objects. Use a new --store directory; this is a research format,
+not a migration or a default storage backend. Small objects currently pay a
+full-page write cost. See docs/research/033-paged-document-storage.md in the
+source repository for measured boundaries and limitations.
+"""
+HYBRID_STORAGE = """# Selected inline/page storage experiment
+
+Objects up to 16 KiB use the existing PUREB001 file format. Larger objects
+use PUREP001 page roots and append-only data generations. The current flat
+binary cannot read PUREP001 objects, so use a new --store directory for this
+preview. This is a selected research build, not the default backend or a
+bidirectional migration path. It requires a Linux host with 16 KiB OS pages.
+See docs/research/034-hybrid-document-storage.md in the source repository.
+"""
+
+RUN = """# Pure document preview — Linux native build
+
+This archive includes the CLI and editable editor.pure program. HTTP, TLS peer,
+and GTK hosts are included only when listed in manifest.json. Shared system libraries are not bundled;
+manifest.json records every binary's direct dependencies.
+
+From this directory, try a local edit:
+
+    printf 'hello Pure\\n' > /tmp/pure-insert.txt
+    bin/pure-edit --program share/editor.pure --store /tmp/pure-preview-store \\
+      edit note 0 0 0 /tmp/pure-insert.txt
+    bin/pure-edit --program share/editor.pure --store /tmp/pure-preview-store show note
+
+When selected, the same program and store can be opened with the GTK window:
+
+    bin/pure-edit-window --program share/editor.pure \\
+      --store /tmp/pure-preview-store note
+
+When selected, the source-loaded terminal editor uses the same program and
+store without GTK. Edit share/editor.pure, then start a new TUI session to
+load the changed definition:
+
+    bin/pure-edit-tui --program share/editor.pure \\
+      --store /tmp/pure-preview-store note
+
+When selected, the TLS peer needs your own CA and server/client certificates. The server
+certificate needs a DNS SAN matching SERVER_DNS; the client certificate needs
+a DNS SAN matching CLIENT_DNS. This example binds only to loopback:
+
+    bin/pure-document-peer serve 127.0.0.1 9443 share/editor.pure \\
+      /tmp/pure-preview-store CA.pem SERVER.pem SERVER.key CLIENT_DNS
+    bin/pure-document-peer call 127.0.0.1 9443 SERVER_DNS share/editor.pure \\
+      CA.pem CLIENT.pem CLIENT.key show note
+
+If manifest.json lists pure-document-server instead, run:
+
+    bin/pure-document-server serve 127.0.0.1 9443 share/editor.pure \\
+      /tmp/pure-preview-store CA.pem SERVER.pem SERVER.key CLIENT_DNS
+
+That selection omits the native `call` command; connect with an authenticated
+external client using the pure-doc/1 binary protocol.
+
+When selected, the HTTP host is a plaintext compatibility adapter; bind it only to loopback.
+This preview was checked on Linux AArch64. It is not a static or macOS package.
+"""
+RUN_FIXED = """# Pure fixed document runner — Linux AArch64
+
+This artifact runs one build-time prepared editor.pure definition. It has no
+source parser, compiler, GTK or TLS host. Editing a .pure file does not
+change this binary; rebuild it with `make fixed-editor` in the source tree.
+
+    bin/pure-edit-fixed --store STORE edit KEY 0 0 0 INSERT_FILE
+    bin/pure-edit-fixed --store STORE show KEY
+
+When manifest.json lists pure-connections-fixed, it serves the same definition
+and store through a loopback HTTP compatibility boundary:
+
+    bin/pure-connections-fixed --store STORE --port 8767
+
+When manifest.json lists pure-edit-tui-fixed, it opens a native terminal
+editor for the same stored document. Ctrl+S saves, Ctrl+O enters another key,
+Ctrl+L browses stored keys (type a prefix to jump), Ctrl+R reloads, and
+Ctrl+Q quits:
+
+    bin/pure-edit-tui-fixed --store STORE note
+
+The external storage files use the default PUREB001 format. The ELF binary
+needs the host's libc and is not a static or macOS package.
+"""
+
+
+def dependencies(path):
+    header = subprocess.check_output(['readelf', '-h', str(path)], text=True)
+    if not re.search(r'Machine:\s+AArch64\s*$', header, re.MULTILINE):
+        raise ValueError(f'{path} is not an AArch64 ELF')
+    output = subprocess.check_output(['readelf', '-d', str(path)], text=True)
+    return sorted(NEEDED.findall(output))
+
+
+def reviewed_budget(build, name, selected=()):
+    command = ['python3', str(HERE / 'check_profile.py'), '--build', str(build),
+               '--budget', str(HERE / name)]
+    for binary in selected:
+        command.extend(['--only', binary])
+    result = subprocess.run(command, capture_output=True, text=True)
+    if result.returncode:
+        raise ValueError(f'{name} failed: {result.stdout or result.stderr}')
+    checked = json.loads(result.stdout)
+    return {'profile': checked['profile'], 'budget_sha256': checked['budget_sha256'],
+            'checked_binaries': sorted(checked['binaries'])}
+
+
+def reviewed_memory(build, fixed=False):
+    result = subprocess.run(['python3', str(HERE / 'check_micro_memory.py'),
+                             '--build', str(build), *(['--fixed'] if fixed else [])],
+                            capture_output=True, text=True)
+    if result.returncode:
+        raise ValueError(f'memory-budget.json failed: {result.stdout or result.stderr}')
+    checked = json.loads(result.stdout)
+    return {'profile': checked['profile'], 'budget_sha256': checked['budget_sha256']}
+
+
+def reviewed_tui_memory(build, source=False):
+    result = subprocess.run(['python3', str(HERE / 'check_tui.py'), '--build', str(build),
+                             *(['--source'] if source else [])],
+                            capture_output=True, text=True)
+    if result.returncode:
+        raise ValueError(f'terminal editor check failed: {result.stdout or result.stderr}')
+    checked = json.loads(result.stdout)
+    return {'profile': checked['profile'],
+            'one_mib_peak_rss_limit_kib': checked['one_mib_peak_rss_limit_kib'],
+            'binary_sha256': checked['binary_sha256']}
+
+
+def add_bytes(archive, name, content, mode):
+    info = tarfile.TarInfo(ROOT_NAME + '/' + name)
+    info.size = len(content)
+    info.mode = mode
+    info.mtime = 0
+    info.uid = info.gid = 0
+    info.uname = info.gname = ''
+    archive.addfile(info, io.BytesIO(content))
+
+
+def without_debug(source, compact=False):
+    # The release build retains symbols for local debugging; the runnable
+    # fixed package needs neither DWARF nor unused symbols. Never mutate the
+    # build used by size checks or source/authoring package symbols.
+    with tempfile.TemporaryDirectory(prefix='pure-package-strip-') as directory:
+        destination = Path(directory) / source.name
+        destination.write_bytes(source.read_bytes())
+        subprocess.run(['strip', '--strip-unneeded' if compact else '--strip-debug',
+                        str(destination)], check=True)
+        if dependencies(destination) != dependencies(source):
+            raise ValueError(f'{source}: stripping changed direct dependencies')
+        return destination.read_bytes()
+
+
+def verify_storage(binary, mode):
+    with tempfile.TemporaryDirectory(prefix='pure-package-store-probe-') as directory:
+        root = Path(directory)
+        payload = root / 'insert'
+        payload.write_bytes(b'X')
+        result = subprocess.run([str(binary), '--program', str(HERE / 'examples/editor.pure'),
+                                 '--store', str(root / 'store'), 'edit', 'probe', '0', '0', '0',
+                                 str(payload)], capture_output=True, timeout=15)
+        expected = b'PUREB001' if mode == 'hybrid' else b'PUREP001'
+        if result.returncode or (root / 'store/obj-probe').read_bytes()[:8] != expected:
+            raise ValueError(f'{binary} is not a working {mode} CLI: {result.stderr[-300:]}')
+        if mode == 'hybrid':
+            payload.write_bytes(b'X' * (16 * 1024 + 1))
+            result = subprocess.run([str(binary), '--program', str(HERE / 'examples/editor.pure'),
+                                     '--store', str(root / 'store'), 'edit', 'large', '0', '0', '0',
+                                     str(payload)], capture_output=True, timeout=15)
+            if result.returncode or (root / 'store/obj-large').read_bytes()[:8] != b'PUREP001':
+                raise ValueError(f'{binary} does not select PUREP001 for large objects')
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--build', type=Path, default=HERE / 'build')
+    parser.add_argument('--output', type=Path, required=True)
+    parser.add_argument('--with-http', action='store_true')
+    parser.add_argument('--with-tui', action='store_true')
+    peer = parser.add_mutually_exclusive_group()
+    peer.add_argument('--with-peer', action='store_true',
+                      help='include the combined native TLS client/server')
+    peer.add_argument('--with-peer-server', action='store_true',
+                      help='include only the authenticated TLS server')
+    parser.add_argument('--with-window', action='store_true')
+    parser.add_argument('--fixed', action='store_true',
+                        help='package only the parser-free build-time prepared editor')
+    storage = parser.add_mutually_exclusive_group()
+    storage.add_argument('--paged-storage', action='store_true',
+                         help='package the historical PUREP001-only storage backend')
+    storage.add_argument('--hybrid-storage', action='store_true',
+                         help='package the selected PUREB001/PUREP001 storage experiment')
+    args = parser.parse_args()
+    if platform.system() != 'Linux' or platform.machine() != 'aarch64':
+        parser.error('this reviewed package profile is Linux AArch64 only')
+    if args.output.exists():
+        parser.error('output already exists')
+    if args.fixed and (args.with_peer or args.with_peer_server or args.with_window or
+                       args.paged_storage or args.hybrid_storage):
+        parser.error('--fixed selects only the flat-store CLI and optional HTTP/TUI hosts')
+    if args.with_tui and args.paged_storage:
+        parser.error('--with-tui supports the default or selected hybrid store')
+    if (args.with_peer or args.with_peer_server) and (args.paged_storage or args.hybrid_storage):
+        parser.error('the document TLS peer currently uses the default flat store')
+    if args.fixed:
+        built = subprocess.run(['make', '-C', str(HERE), f'BUILD={args.build.resolve()}',
+                                'FIXED_PROGRAM=examples/editor.pure', 'fixed-editor',
+                                *(['fixed-server'] if args.with_http else []),
+                                *(['fixed-tui'] if args.with_tui else [])],
+                               capture_output=True, text=True)
+        if built.returncode:
+            parser.error(f'could not prepare the fixed editor: {built.stderr[-500:]}')
+    elif args.with_tui:
+        built = subprocess.run(['make', '-C', str(HERE), f'BUILD={args.build.resolve()}',
+                                *(['STORE=store-paged.c'] if args.hybrid_storage else []),
+                                'editor', 'source-tui'], capture_output=True, text=True)
+        if built.returncode:
+            parser.error(f'could not prepare the source editor: {built.stderr[-500:]}')
+    if args.with_peer_server:
+        built = subprocess.run(['make', '-C', str(HERE), f'BUILD={args.build.resolve()}',
+                                'editor', 'document-server'], capture_output=True, text=True)
+        if built.returncode:
+            parser.error(f'could not prepare the server-only peer: {built.stderr[-500:]}')
+    if args.paged_storage or args.hybrid_storage:
+        verify_storage(args.build / 'pure-edit', 'hybrid' if args.hybrid_storage else 'paged')
+    helper = args.build.resolve() / 'measure-peak-rss'
+    built = subprocess.run(['make', '-C', str(HERE), f'BUILD={args.build.resolve()}',
+                            str(helper)], capture_output=True, text=True)
+    if built.returncode:
+        parser.error(f'could not prepare the RSS helper: {built.stderr[-500:]}')
+    binaries = ['pure-edit-fixed' if args.fixed else 'pure-edit']
+    if args.with_http:
+        binaries.append('pure-connections-fixed' if args.fixed else 'pure-connections')
+    if args.with_tui:
+        binaries.append('pure-edit-tui-fixed' if args.fixed else 'pure-edit-tui')
+    if args.with_peer:
+        binaries.append('pure-document-peer')
+    if args.with_peer_server:
+        binaries.append('pure-document-server')
+    if args.with_window:
+        binaries.append('pure-edit-window')
+    entries = {}
+    manifest_files = {}
+    for binary in binaries:
+        source = args.build / binary
+        content = without_debug(source, args.fixed)
+        if content[:4] != b'\x7fELF':
+            parser.error(f'{source} is not an ELF binary')
+        name = 'bin/' + binary
+        entries[name] = (content, 0o755)
+        manifest_files[name] = {'sha256': hashlib.sha256(content).hexdigest(),
+                                'bytes': len(content), 'needed': dependencies(source)}
+    program = (HERE / 'examples/editor.pure').read_bytes()
+    if not args.fixed:
+        entries['share/editor.pure'] = (program, 0o644)
+        manifest_files['share/editor.pure'] = {'sha256': hashlib.sha256(program).hexdigest(),
+                                               'bytes': len(program)}
+    entries['RUN.md'] = ((RUN_FIXED if args.fixed else RUN).encode(), 0o644)
+    manifest_files['RUN.md'] = {'sha256': hashlib.sha256(entries['RUN.md'][0]).hexdigest(),
+                                'bytes': len(entries['RUN.md'][0])}
+    if args.paged_storage or args.hybrid_storage:
+        entries['STORAGE.md'] = ((HYBRID_STORAGE if args.hybrid_storage else PAGED_STORAGE).encode(), 0o644)
+        manifest_files['STORAGE.md'] = {'sha256': hashlib.sha256(entries['STORAGE.md'][0]).hexdigest(),
+                                        'bytes': len(entries['STORAGE.md'][0])}
+    budgets = [reviewed_budget(args.build,
+                               'fixed-budget.json' if args.fixed else
+                               'hybrid-budget.json' if args.hybrid_storage else
+                               'paged-budget.json' if args.paged_storage else 'profile-budget.json',
+                               ['pure-edit-fixed'] +
+                               (['pure-connections-fixed'] if args.with_http else []) if args.fixed else
+                               ['pure-edit'] + (['pure-connections'] if args.with_http else []))]
+    if args.with_peer:
+        budgets.append(reviewed_budget(args.build, 'peer-budget.json'))
+    if args.with_peer_server:
+        budgets.append(reviewed_budget(args.build, 'peer-server-budget.json'))
+    if args.with_tui:
+        budgets.append(reviewed_budget(args.build,
+                                       'tui-budget.json' if args.fixed else
+                                       'hybrid-tui-budget.json' if args.hybrid_storage else
+                                       'source-tui-budget.json'))
+    if args.with_window:
+        budgets.append(reviewed_budget(args.build, 'window-budget.json'))
+    manifest = {'format': 'pure-document-preview-1', 'platform': 'linux-aarch64',
+                'selected_hosts': binaries, 'shared_libraries_bundled': False,
+                'reviewed_budgets': budgets, 'reviewed_memory': reviewed_memory(args.build, args.fixed),
+                'files': manifest_files}
+    if args.fixed:
+        manifest['program_mode'] = 'build-time-prepared'
+        manifest['source_sha256'] = hashlib.sha256(program).hexdigest()
+    if args.with_tui:
+        manifest['reviewed_tui_memory'] = reviewed_tui_memory(args.build, not args.fixed)
+    if args.with_window:
+        manifest['features'] = ['native-document-key-switch']
+    if args.paged_storage:
+        manifest['storage_profile'] = 'PUREP001-16k-page-research'
+    if args.hybrid_storage:
+        manifest['storage_profile'] = 'PUREB001+PUREP001-hybrid-16k-research'
+    maximum = None
+    package_profile = None
+    if args.fixed:
+        budget_path = HERE / 'fixed-package-budget.json'
+        package_budget = json.loads(budget_path.read_text())
+        package_profile = 'cli' + ('_http' if args.with_http else '') + ('_tui' if args.with_tui else '')
+        maximum = package_budget['max_archive_bytes'][package_profile]
+        manifest['reviewed_package_budget'] = {
+            'profile': package_budget['profile'], 'selection': package_profile,
+            'max_archive_bytes': maximum,
+            'budget_sha256': hashlib.sha256(budget_path.read_bytes()).hexdigest()}
+    elif (args.with_peer or args.with_peer_server) and not (
+            args.with_http or args.with_tui or args.with_window or
+            args.paged_storage or args.hybrid_storage):
+        budget_path = HERE / 'peer-package-budget.json'
+        package_budget = json.loads(budget_path.read_text())
+        package_profile = 'server_only' if args.with_peer_server else 'combined'
+        maximum = package_budget['max_archive_bytes'][package_profile]
+        manifest['reviewed_package_budget'] = {
+            'profile': package_budget['profile'], 'selection': package_profile,
+            'max_archive_bytes': maximum,
+            'budget_sha256': hashlib.sha256(budget_path.read_bytes()).hexdigest()}
+    entries['manifest.json'] = ((json.dumps(manifest, indent=2, sort_keys=True) + '\n').encode(), 0o644)
+    stream = io.BytesIO()
+    with gzip.GzipFile(filename='', mode='wb', fileobj=stream, mtime=0) as compressed:
+        with tarfile.open(fileobj=compressed, mode='w') as archive:
+            for name in sorted(entries):
+                content, mode = entries[name]
+                add_bytes(archive, name, content, mode)
+    packed = stream.getvalue()
+    if maximum is not None and len(packed) > maximum:
+        parser.error(f'{package_profile} archive grew to {len(packed)} bytes '
+                     f'(reviewed maximum {maximum}); no package was published')
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with args.output.open('xb') as output:
+        output.write(packed)
+    print(json.dumps({'archive': str(args.output.resolve()),
+                      'sha256': hashlib.sha256(packed).hexdigest(),
+                      'bytes': len(packed),
+                      'selected_hosts': binaries}, sort_keys=True))
+
+
+if __name__ == '__main__':
+    main()