From 888ecd59f1608b02b890dbc721b0f0d6e3104508 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 08:48:51 -0500 Subject: [PATCH 01/31] Add offline format-5 roll and automatic recovery script Co-Authored-By: Codex --- deployment/scripts/roll-to-format5.sh | 265 ++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100755 deployment/scripts/roll-to-format5.sh diff --git a/deployment/scripts/roll-to-format5.sh b/deployment/scripts/roll-to-format5.sh new file mode 100755 index 0000000000..d254c4ce42 --- /dev/null +++ b/deployment/scripts/roll-to-format5.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +# Offline bundle replacement with a retained format-4 recovery path. +set -euo pipefail +umask 077 + +usage() { + echo 'Usage: roll-to-format5.sh UNIT BUNDLE_BASE STAGED CONFIG STORE READY_URL PEER LEDGER HOLDER TOKEN CONVERT_MINUTES READY_SECONDS [HOME]' >&2 + exit 2 +} +[[ $# == 12 || $# == 13 ]] || usage +unit=$1 base=$2 staged=$3 config=$4 root=$5 ready_url=$6 peer=$7 +ledger=$8 holder=$9 token=${10} minutes=${11} ready_seconds=${12} +node_home=${13:-/root} +[[ $unit != -* && $unit != */* && $unit == *.service ]] || usage +[[ $holder =~ ^[a-zA-Z0-9_.-]+$ && $token =~ ^[a-zA-Z0-9_.-]+$ ]] || usage +[[ $peer =~ ^[a-zA-Z0-9]+$ ]] || usage +[[ $minutes =~ ^[1-9][0-9]{0,4}$ && $ready_seconds =~ ^[1-9][0-9]{0,5}$ ]] || usage +[[ $ready_url == http://* || $ready_url == https://* ]] || usage +[[ $EUID == 0 ]] || { echo 'Root is required.' >&2; exit 2; } +# Canonical paths make rename and containment checks independent of spelling. +for path in "$base" "$staged" "$config" "$root" "$ledger" "$node_home"; do + [[ $path == /* && $path != *$'\n'* ]] || usage +done +base=$(realpath -e -- "$base") +staged=$(realpath -m -- "$staged") +root=$(realpath -e -- "$root") +config=$(realpath -e -- "$config") +live=$base/sdn-bundle +stamp=$(date -u +%Y%m%dT%H%M%S)-$$ +old=$base/sdn-bundle.old-$stamp +failed=$base/sdn-bundle.failed-$stamp +backup=${root}.roll-backup-$stamp +stage=PRECHECK stopped=0 conversion_pid= done_ok=0 +side_files=(dataset-catalogs dataset-feed-head-sync control.flatsqldb + fsql4-migrate.json source-metrics.db storefront.db peers.db + trusted-import-lanes.json homepage.json) +# Preserve SQLite companions when a stopped database retains them. +for name in source-metrics.db storefront.db peers.db; do + side_files+=("$name-wal" "$name-shm" "$name-journal") +done + +log() { + local line + line="$(date -u +%Y-%m-%dT%H:%M:%SZ) $holder $1 token=$token ${2:-}" + printf '%s\n' "$line" >> "$ledger" + printf '%s\n' "$line" || true +} +fail() { log FAILED "step=$stage reason=$1"; exit 1; } +exists() { [[ -e $1 || -L $1 ]]; } + +# Parse systemctl's quoted words without evaluating shell code or printing values. +# Unsupported escape spellings fail closed before the unit stops. +unit_env=() +parse_environment() { + local input=$1 word= char quote= escaped=0 i + for ((i=0; i<${#input}; i++)); do + char=${input:i:1} + if (( escaped )); then + case $char in + ' '|\"|\'|\\|'$'|'`') word+=$char ;; + *) return 1 ;; + esac + escaped=0 + elif [[ $char == \\ ]]; then + escaped=1 + elif [[ -n $quote ]]; then + if [[ $char == "$quote" ]]; then quote=; else word+=$char; fi + elif [[ $char == '"' || $char == "'" ]]; then + quote=$char + elif [[ $char == ' ' ]]; then + if [[ -n $word ]]; then unit_env+=("$word"); word=; fi + else + word+=$char + fi + done + [[ -z $quote && $escaped == 0 ]] || return 1 + [[ -z $word ]] || unit_env+=("$word") + for word in "${unit_env[@]}"; do + [[ $word =~ ^[a-zA-Z_][a-zA-Z0-9_]*= ]] || return 1 + done +} +stop_unit() { + local end=$((SECONDS + 90)) pid state + systemctl stop --no-block "$unit" || return 1 + while (( SECONDS < end )); do + pid=$(systemctl show -p MainPID --value "$unit") || return 1 + state=$(systemctl show -p ActiveState --value "$unit") || return 1 + if [[ $pid == 0 && ( $state == inactive || $state == failed ) ]]; then return 0; fi + sleep 1 + done + return 1 +} +wait_ready() { + local end=$((SECONDS + ready_seconds)) status left + while (( SECONDS < end )); do + left=$((end - SECONDS)) + (( left <= 5 )) || left=5 + status=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --max-time "$left" -- "$ready_url") || status=000 + if [[ $status == 200 ]] && systemctl is-active --quiet "$unit"; then return 0; fi + sleep 1 + done + return 1 +} +check_running_peer() { + local invocation lines line found= + invocation=$(systemctl show -p InvocationID --value "$unit") || return 1 + [[ $invocation =~ ^[a-f0-9]{32}$ ]] || return 1 + lines=$(journalctl --no-pager -o cat "_SYSTEMD_INVOCATION_ID=$invocation" \ + --grep='Peer ID: ') || return 1 + while IFS= read -r line; do + if [[ $line =~ Peer\ ID:\ ([a-zA-Z0-9]+) ]]; then found=${BASH_REMATCH[1]}; fi + done <<< "$lines" + [[ $found == "$peer" ]] +} +stop_conversion() { + [[ -n $conversion_pid ]] || return 0 + if kill -0 "$conversion_pid" 2>/dev/null; then + # Inspect only this child's command line before signaling its exact PID. + local arg matched=0 + while IFS= read -r -d '' arg; do + [[ $arg != store-migrate ]] || matched=1 + done < "/proc/$conversion_pid/cmdline" + (( matched )) || return 1 + kill -KILL "$conversion_pid" || return 1 + fi + wait "$conversion_pid" 2>/dev/null || true + conversion_pid= +} +rollback() { + local after=0 name + stage=STOP_CONVERSION + stop_conversion || return 1 + exists "$root/fsql5" && after=1 + log ROLLBACK "after_switch=$after" || return 1 + stage=STOP_FOR_ROLLBACK + stop_unit || return 1 + if (( after )); then + stage=RESTORE_STORE + [[ -d $backup && -f $root/fsql4/rollback-auxiliary.flatsqlmeta ]] || return 1 + mv -T -- "$root/fsql5" "$root/fsql5.failed-$stamp" || return 1 + cp -p -- "$root/fsql4/rollback-auxiliary.flatsqlmeta" "$root/auxiliary.flatsqlmeta" || return 1 + stage=RESTORE_SIDE_FILES + mkdir -- "$backup/changed" || return 1 + for name in "${side_files[@]}"; do + if exists "$root/$name"; then + mv -T -- "$root/$name" "$backup/changed/$name" || return 1 + fi + if exists "$backup/$name"; then + cp -a -- "$backup/$name" "$root/$name" || return 1 + fi + done + fi + stage=RESTORE_BUNDLE + if exists "$old"; then + if exists "$live"; then mv -T -- "$live" "$failed" || return 1; fi + mv -T -- "$old" "$live" || return 1 + fi + stage=START_OLD + systemctl start --no-block "$unit" || return 1 + wait_ready || return 1 + stage=CHECK_OLD_IDENTITY + check_running_peer || return 1 + if (( after )); then + log ROLLED_BACK_AFTER_SWITCH "reason=$reason" || return 1 + else + log ROLLED_BACK_BEFORE_SWITCH "reason=$reason" || return 1 + fi +} +finish() { + local status=$? failed_step=$stage + trap - EXIT + trap '' TERM HUP INT PIPE + (( done_ok )) && exit 0 + reason=${reason:-"step=$failed_step exit=$status"} + if (( stopped )); then + if ! rollback; then + log ROLLBACK_FAILED "step=$stage reason=$reason" || true + fi + else + log PRECHECK_FAILED "$reason" || true + fi + exit 1 +} +trap finish EXIT +trap 'reason=SIGTERM; exit 143' TERM +trap 'reason=SIGHUP; exit 129' HUP +trap 'reason=SIGINT; exit 130' INT + +log PRECHECK 'begin' +[[ $(systemctl show -p LoadState --value "$unit") == loaded ]] || fail unit_missing +systemctl is-active --quiet "$unit" || fail unit_inactive +user=$(systemctl show -p User --value "$unit") +[[ -z $user || $user == root ]] || fail unit_not_root +[[ -z $(systemctl show -p EnvironmentFiles --value "$unit") ]] || fail environment_files_not_supported +parse_environment "$(systemctl show -p Environment --value "$unit")" || fail environment_encoding +home_override=() +[[ $# == 12 ]] || home_override=("HOME=$node_home") +node_command=(env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin HOME="$node_home" "${unit_env[@]}" "${home_override[@]}") +[[ -d $live && ! -L $live && -d $staged && -f $config ]] || fail bundle_or_config_missing +[[ $staged != "$live" && $staged != "$live/"* && $live != "$staged/"* ]] || fail overlapping_bundles +[[ $root != "$base" && $root != "$base/"* && $base != "$root/"* ]] || fail overlapping_store +[[ $(stat -c %d "$staged") == "$(stat -c %d "$base")" ]] || fail different_filesystems +for path in "$old" "$failed" "$backup" "$root/fsql5.failed-$stamp"; do + ! exists "$path" || fail recovery_path_exists +done +[[ -d $root/fsql4 && ! -L $root/fsql4 ]] || fail format4_missing +! exists "$root/fsql5" || fail format5_exists +[[ -f $root/auxiliary.flatsqlmeta && ! -L $root/auxiliary.flatsqlmeta ]] || fail auxiliary_missing +"${node_command[@]}" "$staged/bin/spacedatanetwork" version >/dev/null 2>&1 || fail staged_version +identity=$("${node_command[@]}" "$staged/bin/spacedatanetwork" show-identity -c "$config" 2>/dev/null) || fail identity_unreadable +[[ $identity == "$peer" ]] || fail identity_mismatch +check_running_peer || fail running_identity_mismatch +bytes=$(du -sb -- "$root/fsql4"); bytes=${bytes%%$'\t'*} +aux=$(stat -c %s -- "$root/auxiliary.flatsqlmeta") +free=$(df -B1 --output=avail -- "$root" | tail -n 1); free=${free// /} +need=$((3 * bytes + aux + 1073741824 + 5000000000)) +log DISK "required_bytes=$need available_bytes=$free" +(( free >= need )) || fail disk_gate +log PRECHECK_OK +stage=STOP +stopped=1 +stop_unit || fail stop_timeout +log STOPPED +stage=SWAP +mv -T -- "$live" "$old" +mv -T -- "$staged" "$live" +log SWAPPED +stage=BACKUP +mkdir -- "$backup" +for name in "${side_files[@]}"; do + if exists "$root/$name"; then cp -a -- "$root/$name" "$backup/$name"; fi +done +log BACKED_UP +stage=CONVERT +# A child lets the shell handle signals while it supervises the foreground task. +signal_pending= +trap 'signal_pending=SIGTERM' TERM +trap 'signal_pending=SIGHUP' HUP +trap 'signal_pending=SIGINT' INT +"${node_command[@]}" "$live/bin/spacedatanetwork" store-migrate --to 5 --config "$config" --store "$root" & +conversion_pid=$! +trap 'reason=SIGTERM; exit 143' TERM +trap 'reason=SIGHUP; exit 129' HUP +trap 'reason=SIGINT; exit 130' INT +if [[ -n $signal_pending ]]; then reason=$signal_pending; exit 1; fi +log CONVERTING "pid=$conversion_pid budget_minutes=$minutes" +deadline=$((SECONDS + minutes * 60)) +while kill -0 "$conversion_pid" 2>/dev/null; do + (( SECONDS < deadline )) || fail conversion_budget + sleep 1 +done +wait "$conversion_pid" || fail conversion_exit +conversion_pid= +stage=CHECK_SWITCH +marker=$(od -An -tx1 -j4 -N2 -- "$root/fsql5/STORE"); marker=${marker// /} +[[ $marker == 0500 && -d $root/fsql4 && -f $root/fsql4/rollback-auxiliary.flatsqlmeta ]] || fail invalid_switch +log SWITCHED +stage=START_NEW +systemctl start --no-block "$unit" +wait_ready || fail ready_budget +stage=CHECK_NEW_IDENTITY +check_running_peer || fail running_identity_mismatch +log SUCCESS 'format=5' +done_ok=1 From 711002265be209e7894a1bf683490978bf6ddc9a Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 08:53:59 -0500 Subject: [PATCH 02/31] Document format-5 recovery and add an offline systemd rehearsal Preserve stopped side files and restore the retained source after failed conversion or readiness checks. Supply isolated store and release inputs to the systemd rehearsal and retain each result and recovery ledger. Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/Dockerfile | 9 + .../roll-to-format5-rehearsal/README.md | 68 ++++++ .../scripts/roll-to-format5-rehearsal/case.py | 231 ++++++++++++++++++ .../scripts/roll-to-format5-rehearsal/run.py | 125 ++++++++++ deployment/scripts/roll-to-format5.md | 153 ++++++++++++ deployment/scripts/roll-to-format5.sh | 19 +- 6 files changed, 599 insertions(+), 6 deletions(-) create mode 100644 deployment/scripts/roll-to-format5-rehearsal/Dockerfile create mode 100644 deployment/scripts/roll-to-format5-rehearsal/README.md create mode 100644 deployment/scripts/roll-to-format5-rehearsal/case.py create mode 100755 deployment/scripts/roll-to-format5-rehearsal/run.py create mode 100644 deployment/scripts/roll-to-format5.md diff --git a/deployment/scripts/roll-to-format5-rehearsal/Dockerfile b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile new file mode 100644 index 0000000000..71a4f404d9 --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:22.04 +# Supply Ubuntu packages and their dependencies for the selected architecture. +# Image assembly performs no package-repository requests. +COPY packages/ /packages/ +RUN dpkg -i /packages/*.deb && \ + test -x /lib/systemd/systemd && command -v curl && command -v python3 +ENV container=docker +STOPSIGNAL SIGRTMIN+3 +CMD ["/lib/systemd/systemd", "--system", "--unit=multi-user.target"] diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md new file mode 100644 index 0000000000..a94ccf566e --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -0,0 +1,68 @@ +# Offline systemd rehearsal + +This harness is an end-to-end test of `../roll-to-format5.sh`. It requires +Docker with privileged containers, a cgroup mount usable by systemd, Python 3.12 +or later on the driver, and enough disk for independent copies of every case. +The image uses Ubuntu 22.04 for the selected architecture, arm64 by default. + +Provide complete Ubuntu `.deb` packages for `systemd`, `systemd-sysv`, `curl`, +`python3`, and their dependencies through `--packages`. The Dockerfile installs +only those files, with build networking disabled. It does not download packages. +A plain Ubuntu container lacks these programs. Image-build failure is a blocked +rehearsal, not a successful test. The node's old and new release archives and +each release's own checksum file are explicit inputs; checksum mismatch aborts +before store copies or containers are made. + +```bash +python3 run.py \ + --old-archive "$OLD_ARCHIVE" --old-checksums "$OLD_CHECKSUMS" \ + --new-archive "$NEW_ARCHIVE" --new-checksums "$NEW_CHECKSUMS" \ + --packages "$OFFLINE_PACKAGES" --arch arm64 \ + --store "$FIRST_STORE_COPY" --store "$SECOND_STORE_COPY" \ + --work-dir "$NEW_REHEARSAL_DIRECTORY" \ + --conversion-minutes 90 --ready-seconds 300 +``` + +Use published v1.1.0 as OLD and v1.1.5 as NEW for the initial deployment test. +Changing `--new-archive` selects another release, with its corresponding checksum +file supplied through `--new-checksums`. The output directory must not exist. +A selective repeat uses `--case` with one or more case names. Start Docker before +running and quit it after the run. The harness stops and removes only containers +it creates. It retains case directories, logs, hashes, ledgers and results. + +The source stores are read only. Copies omit `keys`, `secrets` and `auth.db`. +Every case starts its old node from a fresh copy, with a generated test identity, +loopback listeners, no host ports and `--network=none`. The new release's engine +is prewarmed before stopping the old node. No runtime permission patch or fake +systemctl is used. A failure to start the old release is recorded as a failed +case and requires inspection; the harness does not replace the release or +silently create an empty store. + +| Case | Required evidence | +| --- | --- | +| forward | Zero exit, SUCCESS, identical per-standard public stats, ready and unchanged identity. A second invocation is refused. | +| interrupted | SIGTERM to the recorded migration PID after the pending directory appears, automatic recovery, ready, unchanged identity and identical fsql4 file hashes. | +| budget | One-minute conversion deadline, automatic recovery and identical fsql4 file hashes. A fast conversion that completes inside the budget fails this injection case. | +| after-switch | A unit pre-start condition rejects the new fsql5 selection. The original bundle restarts after recovery removes that selection. | +| wrong-peer | Wrong expected identity is refused before stopping, with unchanged PID, fsql4 hashes and bundle/store selection. | +| disk | Insufficient free bytes on the supplied filesystem are refused before stopping. This case is explicitly not run when space is sufficient. | +| script-term / script-hup | Signal to the exact roll-script MainPID during conversion invokes automatic recovery. | + +The disk case does not fill a host disk or add a test-only script argument. +Supply a work directory on a genuinely constrained filesystem for that case. +Source-store hashes are taken after the old node first becomes ready. The node +can perform background maintenance; a hash mismatch is reported as failure, +not waived. The public `/api/v1/stats` response supplies schema/count aggregates; +an empty response or a changed count fails rather than claiming preservation. + +`results.json` summarizes case outcomes. Each case contains `driver.exit`, +`roll.exit`, `roll.log`, `ledger.log`, aggregate `counts-*.json`, file hash lists, +`journal.log` and `result.json`. Peak migration RSS is sampled from `/proc`; +conversion duration is sampled between CONVERTING and SWITCHED ledger events. +All times describe Docker on the test machine. Archive digests are recorded in +`archives.json`. Test-generated identity material stays in its case directory; +do not publish it or copy it into a deployment. + +The harness and recovery script require execution on both source copies before +use on a production host. An unavailable package set or incomplete case matrix +means the deployment gate has not passed. diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py new file mode 100644 index 0000000000..d81c4b24a8 --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Run inside an isolated container whose PID 1 is systemd.""" +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import time +import urllib.request + +ROOT = Path('/case') +UNIT = 'rehearsal-node.service' +READY = 'http://127.0.0.1:5001/api/v1/ready' +BIN = ROOT / 'base/sdn-bundle/bin/spacedatanetwork' + + +def command(*args, check=True): + return subprocess.run(args, capture_output=True, text=True, check=check).stdout.strip() + + +def wait_ready(seconds): + end = time.monotonic() + seconds + while time.monotonic() < end: + try: + with urllib.request.urlopen(READY, timeout=min(3, max(0.1, end-time.monotonic()))) as r: + if r.status == 200: + return + except Exception: + time.sleep(1) + raise RuntimeError('ready budget exceeded') + + +def hashes(): + result = {} + for p in sorted((ROOT / 'store/fsql4').rglob('*')): + if p.is_file(): + with p.open('rb') as f: + digest = hashlib.sha256() + for block in iter(lambda: f.read(1024 * 1024), b''): + digest.update(block) + result[str(p.relative_to(ROOT / 'store/fsql4'))] = digest.hexdigest() + return result + + +def counts(): + with urllib.request.urlopen('http://127.0.0.1:5001/api/v1/stats', timeout=60) as r: + data = json.load(r) + # Persist only aggregates, never source rows or records. + result = {s['schema']: s['count'] for s in data['schemas']} + if not result or not sum(result.values()): + raise RuntimeError('public stats has no populated per-standard counts') + return result + + +def peer(): + return command(str(BIN), 'show-identity', '-c', '/case/config.yaml') + + +def tree_size(path): + return sum(p.stat().st_size for p in path.rglob('*') if p.is_file()) if path.exists() else 0 + + +def run(case, minutes, seconds): + if Path('/proc/1/comm').read_text().strip() != 'systemd': + raise RuntimeError('PID 1 is not systemd') + (ROOT / 'home').mkdir() + os.environ['HOME'] = '/case/home' + # All keys belong to this isolated test. The container has no external network. + (ROOT / 'config.yaml').write_text('''mode: full +network: + listen: [/ip4/127.0.0.1/tcp/4001] + enable_relay: false +storage: + path: /case/store + max_size: 200GB + gc_interval: 24h +peers: + enable_dht: false + enable_mdns: false +tor: + enabled: false +admin: + enabled: true + listen_addr: 127.0.0.1:5001 + tls_mode: disabled +setup: + data_path: /case/store +''') + unit = Path('/etc/systemd/system') / UNIT + unit.write_text('''[Unit] +Description=Isolated store rehearsal +[Service] +Type=simple +User=root +Environment=HOME=/case/home +ExecStart=/case/base/sdn-bundle/bin/spacedatanetwork daemon --config /case/config.yaml +Restart=no +TimeoutStopSec=75 +''') + command('systemctl', 'daemon-reload') + command('systemctl', 'start', '--no-block', UNIT) + wait_ready(seconds) + expected = peer() + before = counts() + (ROOT / 'counts-before.json').write_text(json.dumps(before, sort_keys=True) + '\n') + before_hashes = hashes() + (ROOT / 'hashes-before.json').write_text(json.dumps(before_hashes, sort_keys=True) + '\n') + # Populate each release's engine cache before the outage. + with open(ROOT / 'prewarm.log', 'w') as log: + subprocess.run(['/case/base/staged/bin/spacedatanetwork', 'prewarm-aot', '--config', + '/case/config.yaml'], stdout=log, stderr=subprocess.STDOUT, check=True) + if case == 'after-switch': + drop = Path('/etc/systemd/system') / (UNIT + '.d') + drop.mkdir() + (drop / 'fail-new.conf').write_text('''[Service] +ExecStartPre=/bin/sh -c 'test ! -d /case/store/fsql5' +''') + command('systemctl', 'daemon-reload') + if case == 'disk': + # Use a genuinely constrained filesystem supplied as the work directory. + # Never fill the host's disk to manufacture a failure. + vfs = os.statvfs(ROOT / 'store') + available = vfs.f_bavail * vfs.f_frsize + required = 3 * tree_size(ROOT / 'store/fsql4') + (ROOT / 'store/auxiliary.flatsqlmeta').stat().st_size + 1073741824 + 5000000000 + (ROOT / 'disk.json').write_text(json.dumps({'available_bytes': available, 'required_bytes': required}) + '\n') + if available >= required: + return {'status': 'not run', 'reason': 'work filesystem has enough free space; supply a constrained filesystem'} + args = ['/bin/bash', '/case/roll.sh', UNIT, '/case/base', '/case/base/staged', + '/case/config.yaml', '/case/store', READY, + 'WrongExpectedPeer' if case == 'wrong-peer' else expected, + '/case/ledger.log', 'rehearsal', case, '1' if case == 'budget' else str(minutes), + str(seconds), '/case/home'] + baseline_pid = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) + start = time.monotonic() + proc = subprocess.Popen(['systemd-run', '--unit=rehearsal-roll', '--collect', '--wait', '--pipe', *args], + stdout=open(ROOT / 'roll.log', 'w'), stderr=subprocess.STDOUT) + injected = False + peak_kib = 0 + migration_start = None + migration_end = None + while proc.poll() is None: + ledger = (ROOT / 'ledger.log').read_text() if (ROOT / 'ledger.log').exists() else '' + match = re.search(r' CONVERTING .*pid=(\d+)', ledger) + if match: + pid = int(match[1]) + migration_start = migration_start or time.monotonic() + stat = Path('/proc') / str(pid) / 'status' + if stat.exists(): + rss = re.search(r'^VmRSS:\s+(\d+)', stat.read_text(), re.M) + if rss: + peak_kib = max(peak_kib, int(rss[1])) + pending = ROOT / 'store/.store-format5-pending' + if not injected and pending.exists() and time.monotonic() - migration_start >= 10: + if case in ('interrupted', 'script-term', 'script-hup'): + if case == 'interrupted': + command_line = (Path('/proc') / str(pid) / 'cmdline').read_bytes().split(b'\0') + if b'store-migrate' not in command_line: + raise RuntimeError('conversion PID no longer matches') + os.kill(pid, signal.SIGTERM) + else: + script_pid = int(command('systemctl', 'show', '-p', 'MainPID', '--value', 'rehearsal-roll')) + command_line = (Path('/proc') / str(script_pid) / 'cmdline').read_bytes() + if b'/case/roll.sh' not in command_line: + raise RuntimeError('script PID no longer matches') + os.kill(script_pid, signal.SIGTERM if case == 'script-term' else signal.SIGHUP) + (ROOT / 'injected.json').write_text(json.dumps({'case': case, 'conversion_pid': pid, 'seconds': time.monotonic()-start}) + '\n') + injected = True + if ' SWITCHED ' in ledger and migration_end is None: + migration_end = time.monotonic() + time.sleep(0.25) + code = proc.wait() + (ROOT / 'roll.exit').write_text(str(code) + '\n') + ledger = (ROOT / 'ledger.log').read_text() + wait_ready(seconds) + if peer() != expected: + raise RuntimeError('identity changed') + after = counts() + (ROOT / 'counts-after.json').write_text(json.dumps(after, sort_keys=True) + '\n') + if before != after: + raise RuntimeError('per-standard counts changed') + if case == 'forward': + if code != 0 or ' SUCCESS ' not in ledger: + raise RuntimeError('forward conversion failed') + with open(ROOT / 'second-run.log', 'w') as log: + second = subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode + (ROOT / 'second-run.exit').write_text(str(second) + '\n') + if second == 0: + raise RuntimeError('second success run was not refused') + elif case in ('wrong-peer', 'disk'): + if code == 0 or ' STOPPED ' in ledger or ' PRECHECK_FAILED ' not in ledger: + raise RuntimeError('precheck did not refuse before stop') + if command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) != baseline_pid: + raise RuntimeError('precheck changed the running daemon') + if before_hashes != hashes(): + raise RuntimeError('precheck changed format 4') + if not (ROOT / 'base/staged').exists() or (ROOT / 'store/fsql5').exists(): + raise RuntimeError('precheck changed bundle or store selection') + else: + event = 'ROLLED_BACK_AFTER_SWITCH' if case == 'after-switch' else 'ROLLED_BACK_BEFORE_SWITCH' + if code == 0 or (' ' + event + ' ') not in ledger: + raise RuntimeError('expected automatic rollback did not complete') + if case in ('interrupted', 'script-term', 'script-hup') and not injected: + raise RuntimeError('failure was not injected during conversion') + if case == 'budget' and 'reason=conversion_budget' not in ledger: + raise RuntimeError('conversion did not reach its budget') + after_hashes = hashes() + (ROOT / 'hashes-after.json').write_text(json.dumps(after_hashes, sort_keys=True) + '\n') + if case != 'after-switch' and before_hashes != after_hashes: + raise RuntimeError('format-4 file hashes changed') + with open(ROOT / 'second-run.log', 'w') as log: + second = subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode + (ROOT / 'second-run.exit').write_text(str(second) + '\n') + if second == 0: + raise RuntimeError('second rollback run was not refused') + return {'status': 'passed', 'roll_exit': code, 'seconds': round(time.monotonic()-start, 2), + 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, + 'peak_conversion_rss_kib_sampled': peak_kib, + 'fsql4_bytes': tree_size(ROOT / 'store/fsql4'), + 'fsql5_bytes': tree_size(ROOT / 'store/fsql5'), 'counts': after} + + +if __name__ == '__main__': + try: + result = run(sys.argv[1], int(sys.argv[2]), int(sys.argv[3])) + except Exception as error: + result = {'status': 'failed', 'reason': str(error)} + (ROOT / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + sys.exit(0 if result['status'] == 'passed' else 1) diff --git a/deployment/scripts/roll-to-format5-rehearsal/run.py b/deployment/scripts/roll-to-format5-rehearsal/run.py new file mode 100755 index 0000000000..177c776aaa --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/run.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Offline systemd rehearsal. Inputs and retained evidence are explicit paths.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile + +CASES = ('forward', 'interrupted', 'budget', 'after-switch', 'wrong-peer', + 'disk', 'script-term', 'script-hup') + + +def verify(archive, checksums): + digest = hashlib.file_digest(open(archive, 'rb'), 'sha256').hexdigest() + expected = [line.split()[0] for line in checksums.read_text().splitlines() + if len(line.split()) == 2 and line.split()[1].lstrip('*') == archive.name] + if expected != [digest]: + raise ValueError('release checksum mismatch: ' + archive.name) + return {'archive': archive.name, 'sha256': digest} + + +def unpack(archive, destination): + destination.mkdir(parents=True) + with tarfile.open(archive) as bundle: + members = bundle.getmembers() + tops = {m.name.split('/')[0] for m in members if m.name} + if len(tops) != 1: + raise ValueError('archive needs one top-level bundle directory') + bundle.extractall(destination.parent / (destination.name + '.unpack'), filter='data') + source = destination.parent / (destination.name + '.unpack') / tops.pop() + for child in source.iterdir(): + shutil.move(str(child), destination / child.name) + source.rmdir() + source.parent.rmdir() + if not (destination / 'bin/spacedatanetwork').is_file(): + raise ValueError('archive has no bundle launcher') + + +def ignore_private(directory, names): + return [n for n in names if n in ('keys', 'secrets', 'auth.db') or 'udl' in n.lower()] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + for name in ('old-archive', 'new-archive', 'old-checksums', 'new-checksums', + 'packages', 'work-dir'): + ap.add_argument('--' + name, required=True, type=Path) + ap.add_argument('--store', required=True, action='append', type=Path) + ap.add_argument('--arch', choices=('arm64', 'amd64'), default='arm64') + ap.add_argument('--case', choices=CASES, action='append') + ap.add_argument('--conversion-minutes', type=int, default=90) + ap.add_argument('--ready-seconds', type=int, default=300) + args = ap.parse_args() + here = Path(__file__).resolve().parent + work = args.work_dir.resolve() + work.mkdir(parents=True, exist_ok=False) + results = [] + active = None + image = 'sdn-format5-rehearsal:' + str(os.getpid()) + try: + verified = [verify(args.old_archive, args.old_checksums), + verify(args.new_archive, args.new_checksums)] + (work / 'archives.json').write_text(json.dumps(verified, indent=2) + '\n') + if not list(args.packages.glob('*.deb')): + raise RuntimeError('offline systemd, systemd-sysv, curl, python3 and dependency packages are required') + context = work / 'image' + context.mkdir() + shutil.copy2(here / 'Dockerfile', context / 'Dockerfile') + shutil.copytree(args.packages, context / 'packages') + with open(work / 'image.log', 'w') as log: + subprocess.run(['docker', 'build', '--network=none', '--platform=linux/' + args.arch, + '-t', image, str(context)], stdout=log, stderr=subprocess.STDOUT, check=True) + for number, source in enumerate(args.store, 1): + for case in args.case or CASES: + directory = work / ('store-' + str(number)) / case + directory.mkdir(parents=True) + shutil.copytree(source, directory / 'store', ignore=ignore_private) + unpack(args.old_archive, directory / 'base/sdn-bundle') + unpack(args.new_archive, directory / 'base/staged') + shutil.copy2(here.parent / 'roll-to-format5.sh', directory / 'roll.sh') + shutil.copy2(here / 'case.py', directory / 'case.py') + active = 'sdn-format5-' + str(os.getpid()) + '-' + str(number) + '-' + case + with open(directory / 'container.log', 'w') as log: + subprocess.run(['docker', 'run', '-d', '--name', active, '--privileged', + '--cgroupns=host', '--network=none', '--platform=linux/' + args.arch, + '--cpus=2', '--memory=6g', '--tmpfs=/run', '--tmpfs=/run/lock', + '--tmpfs=/tmp', '-v', '/sys/fs/cgroup:/sys/fs/cgroup:rw', + '-v', str(directory) + ':/case', image], + stdout=log, stderr=subprocess.STDOUT, check=True) + try: + with open(directory / 'driver.log', 'w') as log: + status = subprocess.run(['docker', 'exec', active, 'python3', '/case/case.py', + case, str(args.conversion_minutes), str(args.ready_seconds)], + stdout=log, stderr=subprocess.STDOUT).returncode + (directory / 'driver.exit').write_text(str(status) + '\n') + result_file = directory / 'result.json' + result = json.loads(result_file.read_text()) if result_file.exists() else { + 'status': 'failed', 'reason': 'driver exited without a result'} + results.append({'store': number, 'case': case, **result}) + finally: + with open(directory / 'journal.log', 'w') as log: + subprocess.run(['docker', 'exec', active, 'journalctl', '--no-pager', '-o', 'short-iso'], + stdout=log, stderr=subprocess.STDOUT) + # Docker addresses only the container created above. + subprocess.run(['docker', 'stop', active], check=True, stdout=subprocess.DEVNULL) + subprocess.run(['docker', 'rm', active], check=True, stdout=subprocess.DEVNULL) + active = None + (work / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + except Exception as error: + (work / 'blocked.txt').write_text(str(error) + '\n') + raise + finally: + # Retain every store and log, including failed cases. + if active: + subprocess.run(['docker', 'stop', active], stdout=subprocess.DEVNULL) + (work / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + return int(any(r['status'] != 'passed' for r in results)) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/deployment/scripts/roll-to-format5.md b/deployment/scripts/roll-to-format5.md new file mode 100644 index 0000000000..efad2863f4 --- /dev/null +++ b/deployment/scripts/roll-to-format5.md @@ -0,0 +1,153 @@ +# Roll a stopped format-4 store to format 5 + +`roll-to-format5.sh` replaces a complete bundle, converts the store, and checks +readiness and identity. A failure after stopping the service triggers automatic +recovery. The script requires Linux, Bash, GNU coreutils, systemd and curl, and +runs as root. The service also runs as root. No schema or record data is printed +by the script. Conversion output comes from the node binary. + +The release-source recovery contract is verified against v1.1.5. End-to-end +validation on production-store copies is required before deployment. The +included rehearsal needs offline systemd packages; syntax checks alone do not +establish recovery safety. + +## Preparation + +1. Acquire the deployment lock with the deployment's exclusive lock procedure. + Record the holder, token and manual-install reason in the deployment ledger. + Keep that lock through recovery or success. The script records the supplied + holder and token; it does not acquire or release an external lock. +2. Download the complete release archive, verify it against the release checksum + file, and unpack it without preserving archive ownership. Verify the manifest, + platform, version and trust roots. Stage it on the same filesystem as the + bundle base. The live folder is named `sdn-bundle`. The store is outside both + bundle directories. Neither bundle is an ancestor of the other. +3. Check the service configuration and store root agree. Check any module catalog + hashes required by the deployment. Run `version`, `show-identity -c CONFIG`, + and `key fingerprint-check` with the service's environment. Never add + `--show-mnemonic`. Record the running daemon's peer ID. +4. Run the staged binary's `prewarm-aot --config CONFIG` as root with the + service's environment and cache HOME. Record the old binary checksum. +5. Ensure the ledger is writable and private. Ensure the ready URL is the node's + unauthenticated ready endpoint and is usable by both releases. A local URL + avoids a proxy's cached response. Ensure the current invocation's journal + contains the node's `Peer ID:` startup line. + +The script reads `systemctl show -p Environment --value` without printing it. +It parses quoted words without evaluating shell code. Unsupported escape +sequences, non-root services and `EnvironmentFile` settings are refused before +the outage. Supply the service's HOME as the final argument when it differs +from the default. Explicit HOME takes precedence over the unit's Environment. +The service's PATH is honored when present; the default is the system PATH. +Service namespaces and filesystem isolation need equivalent command access: +configuration and store paths must identify the same files outside the unit. + +The disk gate requires free bytes of at least: + +``` +3 * du_bytes(fsql4) + bytes(auxiliary.flatsqlmeta) + 1073741824 + 5000000000 +``` + +An auxiliary metadata file is required because this procedure promises its +recovery snapshot. Symbolic links for the live bundle, fsql4 and auxiliary file +are refused. Check all paths and budgets before the outage. + +## Invocation + +Set the variables to the deployment's values, then start one transient service: + +```bash +systemd-run --unit="$ROLL_UNIT" --collect \ + /bin/bash "$SCRIPT" \ + "$NODE_UNIT" "$BUNDLE_BASE" "$STAGED_BUNDLE" "$CONFIG" "$STORE_ROOT" \ + "$READY_URL" "$EXPECTED_PEER" "$LEDGER" "$HOLDER" "$TOKEN" \ + "$CONVERSION_MINUTES" "$READY_SECONDS" "$NODE_HOME" +journalctl -u "$ROLL_UNIT" -f +``` + +The first budget is a positive whole number of minutes; the second is a positive +whole number of seconds. The readiness budget applies separately to the new +node and to recovery. Stop waits are bounded at 90 seconds. Conversion is +supervised as a child of the script; its output remains attached to the script. +At the conversion deadline the script kills and waits for that exact child +before inspecting the switch. It does not leave an independent migration job. +The conversion command includes `--store STORE_ROOT` as well as `--config` to +bind the conversion to the checked root. + +SIGTERM, SIGHUP and SIGINT sent to the script invoke the same recovery path. +SIGKILL, power loss, a kernel failure or the loss of the filesystem prevents +shell traps from running. The transient service detaches the outage from the +operator's login session. Do not stop the entire transient unit to simulate a +signal to the script: systemd can also terminate its children. + +## Ledger + +Every event is appended as one UTC line and also written to standard output: + +``` + token= +``` + +| Event | Meaning | +| --- | --- | +| PRECHECK | Read-only checks started, apart from the ledger and the identity command's permission enforcement. | +| DISK | Required and available byte counts. | +| PRECHECK_OK | Environment, identity, path and space checks passed. | +| STOPPED | MainPID is zero and the unit is inactive or failed. | +| SWAPPED | The old bundle is retained and the staged bundle occupies the live name. | +| BACKED_UP | Stopped-state side files are copied beside the store. | +| CONVERTING | Exact conversion PID and time budget. | +| SWITCHED | Format bytes are `05 00`; fsql4 and its auxiliary snapshot exist. | +| SUCCESS | The new service answers ready and its invocation journal has the expected peer ID. | +| FAILED | A checked step failed; recovery follows if stopping began. | +| PRECHECK_FAILED | No outage began. | +| ROLLBACK | Recovery started; `after_switch` records whether fsql5 exists. | +| ROLLED_BACK_BEFORE_SWITCH | The original bundle is serving format 4 with the expected identity. | +| ROLLED_BACK_AFTER_SWITCH | Format 5 is retained under a failed name, side files and auxiliary metadata are restored, and the original bundle is serving. | +| ROLLBACK_FAILED | Recovery stopped at the named step. Human intervention is required. | + +Argument and path-resolution errors can exit before ledger initialization. +Success exits zero. Refusal, conversion failure and completed recovery exit +nonzero. A failed recovery also exits nonzero. + +## Recovery artifacts + +The original bundle becomes `sdn-bundle.old--`. Recovery moves the +replacement to `sdn-bundle.failed--` and restores the original live +name. Side files are copied into `STORE_ROOT.roll-backup--`: +`dataset-catalogs`, `dataset-feed-head-sync`, `control.flatsqldb`, +`fsql4-migrate.json`, `source-metrics.db`, `storefront.db`, `peers.db`, +`trusted-import-lanes.json` and `homepage.json`, when present. SQLite `-wal`, +`-shm` and `-journal` companions of the three `.db` files are included. +`keys`, `secrets` and `auth.db` are excluded. + +After a switch, recovery moves `fsql5` to `fsql5.failed--` and copies +`fsql4/rollback-auxiliary.flatsqlmeta` over the root auxiliary metadata. Side +files from the failed run are moved to the backup's `changed` folder before +restoring their originals, including original absence. No retained store or +bundle is deleted. Records accepted after conversion are absent from the old +store. Quiesce external ingestion during the deployment window. + +The v1.1.5 converter removes its unpublished candidate when returning an error; +an abruptly terminated converter can leave `.store-format5-pending`. The next +conversion reclaims only a candidate whose ownership marker matches the source. +The script leaves this candidate to the converter. A termination immediately +before the atomic switch can leave the auxiliary rollback snapshot in fsql4. + +A repeat with the original arguments is refused: after success fsql5 exists and +the staged directory is consumed; after recovery the staged directory is also +consumed. A deliberate retry requires a newly staged bundle and fresh checks. + +On `ROLLBACK_FAILED`, retain the lock and all artifacts. Read the named step, +ledger and service journals. Determine which bundle and store the unit selects +before starting it. Resolve the failed operation and complete the documented +recovery sequence manually. Do not delete fsql4, relabel a STORE marker, rerun +the script over partial recovery, or overwrite the preserved bundles. + +## Rehearsal + +See [the offline harness](roll-to-format5-rehearsal/README.md). Its image runs +real systemd as PID 1. It uses isolated copies, generated test identities and no +external network. A passing report includes old/new per-standard counts, file +hashes, exit codes and the ledger for each store and failure case. Docker +measurements describe that environment and do not predict host outage times. diff --git a/deployment/scripts/roll-to-format5.sh b/deployment/scripts/roll-to-format5.sh index d254c4ce42..7109a7c4de 100755 --- a/deployment/scripts/roll-to-format5.sh +++ b/deployment/scripts/roll-to-format5.sh @@ -45,7 +45,7 @@ log() { printf '%s\n' "$line" >> "$ledger" printf '%s\n' "$line" || true } -fail() { log FAILED "step=$stage reason=$1"; exit 1; } +fail() { reason=$1; log FAILED "step=$stage reason=$reason"; exit 1; } exists() { [[ -e $1 || -L $1 ]]; } # Parse systemctl's quoted words without evaluating shell code or printing values. @@ -117,12 +117,16 @@ stop_conversion() { [[ -n $conversion_pid ]] || return 0 if kill -0 "$conversion_pid" 2>/dev/null; then # Inspect only this child's command line before signaling its exact PID. - local arg matched=0 + local arg matched=0 has_args=0 while IFS= read -r -d '' arg; do + has_args=1 [[ $arg != store-migrate ]] || matched=1 done < "/proc/$conversion_pid/cmdline" - (( matched )) || return 1 - kill -KILL "$conversion_pid" || return 1 + if (( matched )); then + kill -KILL "$conversion_pid" 2>/dev/null || ! kill -0 "$conversion_pid" 2>/dev/null || return 1 + elif (( has_args )); then + return 1 + fi fi wait "$conversion_pid" 2>/dev/null || true conversion_pid= @@ -133,8 +137,10 @@ rollback() { stop_conversion || return 1 exists "$root/fsql5" && after=1 log ROLLBACK "after_switch=$after" || return 1 - stage=STOP_FOR_ROLLBACK - stop_unit || return 1 + if exists "$old" || (( after )); then + stage=STOP_FOR_ROLLBACK + stop_unit || return 1 + fi if (( after )); then stage=RESTORE_STORE [[ -d $backup && -f $root/fsql4/rollback-auxiliary.flatsqlmeta ]] || return 1 @@ -200,6 +206,7 @@ node_command=(env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin HOME="$node_home" "${uni [[ -d $live && ! -L $live && -d $staged && -f $config ]] || fail bundle_or_config_missing [[ $staged != "$live" && $staged != "$live/"* && $live != "$staged/"* ]] || fail overlapping_bundles [[ $root != "$base" && $root != "$base/"* && $base != "$root/"* ]] || fail overlapping_store +[[ $root != "$staged" && $root != "$staged/"* && $staged != "$root/"* ]] || fail overlapping_staging [[ $(stat -c %d "$staged") == "$(stat -c %d "$base")" ]] || fail different_filesystems for path in "$old" "$failed" "$backup" "$root/fsql5.failed-$stamp"; do ! exists "$path" || fail recovery_path_exists From 5c0c8c51f855ebdae005b9e1180a2b97bb609a87 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 08:55:32 -0500 Subject: [PATCH 03/31] Clarify interrupted conversion recovery wording Co-Authored-By: Codex --- deployment/scripts/roll-to-format5.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/scripts/roll-to-format5.md b/deployment/scripts/roll-to-format5.md index efad2863f4..b463239d0c 100644 --- a/deployment/scripts/roll-to-format5.md +++ b/deployment/scripts/roll-to-format5.md @@ -129,7 +129,7 @@ bundle is deleted. Records accepted after conversion are absent from the old store. Quiesce external ingestion during the deployment window. The v1.1.5 converter removes its unpublished candidate when returning an error; -an abruptly terminated converter can leave `.store-format5-pending`. The next +an abruptly terminated converter can leave `.store-format5-pending`. A subsequent conversion reclaims only a candidate whose ownership marker matches the source. The script leaves this candidate to the converter. A termination immediately before the atomic switch can leave the auxiliary rollback snapshot in fsql4. From 25834af2af302420cea70170ca6c660613d571cb Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 08:59:50 -0500 Subject: [PATCH 04/31] Install rehearsal dependencies from official Ubuntu archives Build the systemd image with its distribution packages and retain isolated networking for release and store checks. Co-Authored-By: Codex --- .../scripts/roll-to-format5-rehearsal/Dockerfile | 8 +++----- .../scripts/roll-to-format5-rehearsal/README.md | 14 ++++++-------- .../scripts/roll-to-format5-rehearsal/run.py | 9 +++------ deployment/scripts/roll-to-format5.md | 4 ++-- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/Dockerfile b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile index 71a4f404d9..342f53b7a1 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/Dockerfile +++ b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile @@ -1,9 +1,7 @@ FROM ubuntu:22.04 -# Supply Ubuntu packages and their dependencies for the selected architecture. -# Image assembly performs no package-repository requests. -COPY packages/ /packages/ -RUN dpkg -i /packages/*.deb && \ - test -x /lib/systemd/systemd && command -v curl && command -v python3 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + systemd systemd-sysv dbus curl ca-certificates procps python3 && \ + rm -rf /var/lib/apt/lists/* ENV container=docker STOPSIGNAL SIGRTMIN+3 CMD ["/lib/systemd/systemd", "--system", "--unit=multi-user.target"] diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index a94ccf566e..0b0afbe02d 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -1,15 +1,13 @@ -# Offline systemd rehearsal +# Isolated systemd rehearsal This harness is an end-to-end test of `../roll-to-format5.sh`. It requires Docker with privileged containers, a cgroup mount usable by systemd, Python 3.12 or later on the driver, and enough disk for independent copies of every case. The image uses Ubuntu 22.04 for the selected architecture, arm64 by default. -Provide complete Ubuntu `.deb` packages for `systemd`, `systemd-sysv`, `curl`, -`python3`, and their dependencies through `--packages`. The Dockerfile installs -only those files, with build networking disabled. It does not download packages. -A plain Ubuntu container lacks these programs. Image-build failure is a blocked -rehearsal, not a successful test. The node's old and new release archives and +The Dockerfile installs systemd, D-Bus, curl, Python and supporting utilities +from Ubuntu's official package archives. Package downloads occur only during +the image build. Image-build failure is a blocked rehearsal. The node's old and new release archives and each release's own checksum file are explicit inputs; checksum mismatch aborts before store copies or containers are made. @@ -17,7 +15,7 @@ before store copies or containers are made. python3 run.py \ --old-archive "$OLD_ARCHIVE" --old-checksums "$OLD_CHECKSUMS" \ --new-archive "$NEW_ARCHIVE" --new-checksums "$NEW_CHECKSUMS" \ - --packages "$OFFLINE_PACKAGES" --arch arm64 \ + --arch arm64 \ --store "$FIRST_STORE_COPY" --store "$SECOND_STORE_COPY" \ --work-dir "$NEW_REHEARSAL_DIRECTORY" \ --conversion-minutes 90 --ready-seconds 300 @@ -64,5 +62,5 @@ All times describe Docker on the test machine. Archive digests are recorded in do not publish it or copy it into a deployment. The harness and recovery script require execution on both source copies before -use on a production host. An unavailable package set or incomplete case matrix +use on a production host. An image-build failure or incomplete case matrix means the deployment gate has not passed. diff --git a/deployment/scripts/roll-to-format5-rehearsal/run.py b/deployment/scripts/roll-to-format5-rehearsal/run.py index 177c776aaa..a9e1bd6c92 100755 --- a/deployment/scripts/roll-to-format5-rehearsal/run.py +++ b/deployment/scripts/roll-to-format5-rehearsal/run.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Offline systemd rehearsal. Inputs and retained evidence are explicit paths.""" +"""Isolated systemd rehearsal. Inputs and retained evidence are explicit paths.""" import argparse import hashlib import json @@ -47,7 +47,7 @@ def ignore_private(directory, names): def main(): ap = argparse.ArgumentParser(description=__doc__) for name in ('old-archive', 'new-archive', 'old-checksums', 'new-checksums', - 'packages', 'work-dir'): + 'work-dir',): ap.add_argument('--' + name, required=True, type=Path) ap.add_argument('--store', required=True, action='append', type=Path) ap.add_argument('--arch', choices=('arm64', 'amd64'), default='arm64') @@ -65,14 +65,11 @@ def main(): verified = [verify(args.old_archive, args.old_checksums), verify(args.new_archive, args.new_checksums)] (work / 'archives.json').write_text(json.dumps(verified, indent=2) + '\n') - if not list(args.packages.glob('*.deb')): - raise RuntimeError('offline systemd, systemd-sysv, curl, python3 and dependency packages are required') context = work / 'image' context.mkdir() shutil.copy2(here / 'Dockerfile', context / 'Dockerfile') - shutil.copytree(args.packages, context / 'packages') with open(work / 'image.log', 'w') as log: - subprocess.run(['docker', 'build', '--network=none', '--platform=linux/' + args.arch, + subprocess.run(['docker', 'build', '--platform=linux/' + args.arch, '-t', image, str(context)], stdout=log, stderr=subprocess.STDOUT, check=True) for number, source in enumerate(args.store, 1): for case in args.case or CASES: diff --git a/deployment/scripts/roll-to-format5.md b/deployment/scripts/roll-to-format5.md index b463239d0c..7d9d5862e7 100644 --- a/deployment/scripts/roll-to-format5.md +++ b/deployment/scripts/roll-to-format5.md @@ -8,7 +8,7 @@ by the script. Conversion output comes from the node binary. The release-source recovery contract is verified against v1.1.5. End-to-end validation on production-store copies is required before deployment. The -included rehearsal needs offline systemd packages; syntax checks alone do not +included rehearsal installs systemd from official package archives; syntax checks do not establish recovery safety. ## Preparation @@ -146,7 +146,7 @@ the script over partial recovery, or overwrite the preserved bundles. ## Rehearsal -See [the offline harness](roll-to-format5-rehearsal/README.md). Its image runs +See [the rehearsal harness](roll-to-format5-rehearsal/README.md). Its image runs real systemd as PID 1. It uses isolated copies, generated test identities and no external network. A passing report includes old/new per-standard counts, file hashes, exit codes and the ledger for each store and failure case. Docker From a3ccde0ce02e34569563b6643170d4340f915a47 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 09:06:32 -0500 Subject: [PATCH 05/31] Give rehearsal containers stable identity inputs A stable hostname allows the release to generate its isolated test identity. Docker's default hexadecimal hostname is refused by the identity guard. Co-Authored-By: Codex --- deployment/scripts/roll-to-format5-rehearsal/README.md | 4 +++- deployment/scripts/roll-to-format5-rehearsal/run.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 0b0afbe02d..b358af2a2e 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -30,7 +30,9 @@ it creates. It retains case directories, logs, hashes, ledgers and results. The source stores are read only. Copies omit `keys`, `secrets` and `auth.db`. Every case starts its old node from a fresh copy, with a generated test identity, -loopback listeners, no host ports and `--network=none`. The new release's engine +a stable container hostname, loopback listeners, no host ports and +`--network=none`. The stable hostname allows the release to create a durable +test identity without Docker's ephemeral-hostname refusal. The new release's engine is prewarmed before stopping the old node. No runtime permission patch or fake systemctl is used. A failure to start the old release is recorded as a failed case and requires inspection; the harness does not replace the release or diff --git a/deployment/scripts/roll-to-format5-rehearsal/run.py b/deployment/scripts/roll-to-format5-rehearsal/run.py index a9e1bd6c92..c7798223d5 100755 --- a/deployment/scripts/roll-to-format5-rehearsal/run.py +++ b/deployment/scripts/roll-to-format5-rehearsal/run.py @@ -82,7 +82,7 @@ def main(): shutil.copy2(here / 'case.py', directory / 'case.py') active = 'sdn-format5-' + str(os.getpid()) + '-' + str(number) + '-' + case with open(directory / 'container.log', 'w') as log: - subprocess.run(['docker', 'run', '-d', '--name', active, '--privileged', + subprocess.run(['docker', 'run', '-d', '--name', active, '--hostname=sdn-rehearsal', '--privileged', '--cgroupns=host', '--network=none', '--platform=linux/' + args.arch, '--cpus=2', '--memory=6g', '--tmpfs=/run', '--tmpfs=/run/lock', '--tmpfs=/tmp', '-v', '/sys/fs/cgroup:/sys/fs/cgroup:rw', From 5b1f092c4b1192487b35938682309852b8a53074 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 09:10:17 -0500 Subject: [PATCH 06/31] Record the published release baseline failure The v1.1.0 arm64 bundle rejects both format-4 copies. An empty-store control reaches readiness and creates format-5 marker bytes under fsql4. Conversion and recovery remain unverified on these copies. Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index b358af2a2e..4f659e778e 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -50,7 +50,7 @@ silently create an empty store. The disk case does not fill a host disk or add a test-only script argument. Supply a work directory on a genuinely constrained filesystem for that case. -Source-store hashes are taken after the old node first becomes ready. The node +Copied-store hashes are taken after the old node first becomes ready. The node can perform background maintenance; a hash mismatch is reported as failure, not waived. The public `/api/v1/stats` response supplies schema/count aggregates; an empty response or a changed count fails rather than claiming preservation. @@ -66,3 +66,19 @@ do not publish it or copy it into a deployment. The harness and recovery script require execution on both source copies before use on a production host. An image-build failure or incomplete case matrix means the deployment gate has not passed. + +## Release baseline result + +The published v1.1.0 linux/arm64 archive, verified against its release checksum +file, fails to open both supplied format-4 store copies. Startup reports +`P4_E_FORMAT` and `flatsql_init returned status -5` before readiness. The same +archive reaches readiness with an empty store in the same systemd image. +That control creates `fsql4/STORE` with bytes 4 and 5 equal to `05 00`, while +both supplied stores contain `04 00`. The binary reports version 1.1.0. +No store marker is relabeled and no node code is changed. + +The old-node baseline gate has failed. Forward conversion, failure recovery, +repeat invocation and precheck cases have no end-to-end result on these copies. +The v1.1.5 archive checksum passes, but its conversion and recovery behavior +remain unverified. The script requires a passing baseline and complete case +matrix before deployment. From 64d1516131e198a270869d04837ac2d105323cde Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 09:30:02 -0500 Subject: [PATCH 07/31] Rehearse installed binaries in Ubuntu 24.04 containers Pair each store with its verified installed binary and preserve the old and new bundle layouts. Isolate bootstrap traffic to loopback and supply fresh test passwords and engine caches. Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/Dockerfile | 12 +++++++- .../roll-to-format5-rehearsal/README.md | 30 ++++++++++++++----- .../scripts/roll-to-format5-rehearsal/case.py | 27 +++++++++++++++-- .../scripts/roll-to-format5-rehearsal/run.py | 25 ++++++++++++---- 4 files changed, 76 insertions(+), 18 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/Dockerfile b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile index 342f53b7a1..5e4bbbf964 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/Dockerfile +++ b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile @@ -1,7 +1,17 @@ -FROM ubuntu:22.04 +FROM --platform=$BUILDPLATFORM ubuntu:24.04 AS init +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + systemd systemd-sysv dbus && rm -rf /var/lib/apt/lists/* + +FROM ubuntu:24.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ systemd systemd-sysv dbus curl ca-certificates procps python3 && \ rm -rf /var/lib/apt/lists/* +# Native systemd launches emulated release binaries through the kernel handler. +# Keep the target architecture's libraries alongside the native init libraries. +COPY --from=init /usr/lib/ /usr/lib/ +# Container init must not replace the host kernel's binary-format handlers. +RUN ln -sf /dev/null /etc/systemd/system/systemd-binfmt.service && \ + ln -sf /dev/null /etc/systemd/system/proc-sys-fs-binfmt_misc.mount ENV container=docker STOPSIGNAL SIGRTMIN+3 CMD ["/lib/systemd/systemd", "--system", "--unit=multi-user.target"] diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 4f659e778e..6032de2f24 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -3,25 +3,35 @@ This harness is an end-to-end test of `../roll-to-format5.sh`. It requires Docker with privileged containers, a cgroup mount usable by systemd, Python 3.12 or later on the driver, and enough disk for independent copies of every case. -The image uses Ubuntu 22.04 for the selected architecture, arm64 by default. +The image uses Ubuntu 24.04 for the selected node architecture, amd64 by default. +Systemd and its support libraries run on the build platform's native architecture; +node binaries and target runtime libraries retain the selected architecture. +This supports systemd service execution when the emulator rejects the amd64 +systemd executor. Container binary-format management is masked so it cannot +replace the Docker kernel's emulator registrations. The Dockerfile installs systemd, D-Bus, curl, Python and supporting utilities from Ubuntu's official package archives. Package downloads occur only during -the image build. Image-build failure is a blocked rehearsal. The node's old and new release archives and -each release's own checksum file are explicit inputs; checksum mismatch aborts +the image build. Image-build failure is a blocked rehearsal. Each source store's old binary and SHA-256, plus the new release archive and +its release checksum file, are explicit inputs; checksum mismatch aborts before store copies or containers are made. ```bash python3 run.py \ - --old-archive "$OLD_ARCHIVE" --old-checksums "$OLD_CHECKSUMS" \ + --old-binary "$FIRST_OLD_BINARY" --old-sha256 "$FIRST_OLD_SHA256" \ + --old-binary "$SECOND_OLD_BINARY" --old-sha256 "$SECOND_OLD_SHA256" \ --new-archive "$NEW_ARCHIVE" --new-checksums "$NEW_CHECKSUMS" \ - --arch arm64 \ + --arch amd64 \ --store "$FIRST_STORE_COPY" --store "$SECOND_STORE_COPY" \ --work-dir "$NEW_REHEARSAL_DIRECTORY" \ --conversion-minutes 90 --ready-seconds 300 ``` -Use published v1.1.0 as OLD and v1.1.5 as NEW for the initial deployment test. +Use the binary matching each store's installed build as OLD and published +v1.1.5 as NEW. Each old binary has its bundle's `manifest.json` beside it. +The old bundle contains `bin/spacedatanetwork` and that manifest; the new bundle +retains the release archive's launcher and runtime layout. Old binary arguments +and their digests follow the order of the store arguments. Changing `--new-archive` selects another release, with its corresponding checksum file supplied through `--new-checksums`. The output directory must not exist. A selective repeat uses `--case` with one or more case names. Start Docker before @@ -30,9 +40,13 @@ it creates. It retains case directories, logs, hashes, ledgers and results. The source stores are read only. Copies omit `keys`, `secrets` and `auth.db`. Every case starts its old node from a fresh copy, with a generated test identity, -a stable container hostname, loopback listeners, no host ports and +a generated test password, a separate engine cache, a stable container hostname, +loopback listeners, disabled DHT and mDNS, no trusted peers, no host ports and `--network=none`. The stable hostname allows the release to create a durable -test identity without Docker's ephemeral-hostname refusal. The new release's engine +test identity without Docker's ephemeral-hostname refusal. These releases +replace an empty bootstrap list with public defaults. A synthetic peer identity +at unused loopback port 9 suppresses that fallback without naming a real peer. +Docker's network namespace has no external interface. The new release's engine is prewarmed before stopping the old node. No runtime permission patch or fake systemctl is used. A failure to start the old release is recorded as a failed case and requires inspection; the harness does not replace the release or diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index d81c4b24a8..8aae0351ae 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -67,17 +67,36 @@ def run(case, minutes, seconds): if Path('/proc/1/comm').read_text().strip() != 'systemd': raise RuntimeError('PID 1 is not systemd') (ROOT / 'home').mkdir() + (ROOT / 'cache').mkdir() + password = ROOT / 'test-password' + password.write_text(os.urandom(32).hex()) + password.chmod(0o600) + os.environ['SDN_KEY_PASSWORD_FILE'] = str(password) + os.environ['XDG_CACHE_HOME'] = '/case/cache' os.environ['HOME'] = '/case/home' + # Empty bootstrap lists trigger built-in public defaults in these releases. + # A synthetic identity at an unused loopback port prevents that fallback. + alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + value = int.from_bytes(bytes.fromhex('002408011220') + bytes([1]) * 32, 'big') + encoded = '' + while value: + value, digit = divmod(value, 58) + encoded = alphabet[digit] + encoded + isolated_peer = '1' + encoded # All keys belong to this isolated test. The container has no external network. (ROOT / 'config.yaml').write_text('''mode: full network: listen: [/ip4/127.0.0.1/tcp/4001] - enable_relay: false + bootstrap: [/ip4/127.0.0.1/tcp/9/p2p/ISOLATED_PEER] + edge_relays: [] + enable_relay: never storage: path: /case/store max_size: 200GB gc_interval: 24h peers: + trusted_peers: [] + registry_path: /case/empty-peer-registry.json enable_dht: false enable_mdns: false tor: @@ -88,14 +107,14 @@ def run(case, minutes, seconds): tls_mode: disabled setup: data_path: /case/store -''') +'''.replace('ISOLATED_PEER', isolated_peer)) unit = Path('/etc/systemd/system') / UNIT unit.write_text('''[Unit] Description=Isolated store rehearsal [Service] Type=simple User=root -Environment=HOME=/case/home +Environment=HOME=/case/home XDG_CACHE_HOME=/case/cache SDN_KEY_PASSWORD_FILE=/case/test-password ExecStart=/case/base/sdn-bundle/bin/spacedatanetwork daemon --config /case/config.yaml Restart=no TimeoutStopSec=75 @@ -103,6 +122,8 @@ def run(case, minutes, seconds): command('systemctl', 'daemon-reload') command('systemctl', 'start', '--no-block', UNIT) wait_ready(seconds) + command('systemctl', 'is-active', '--quiet', UNIT) + (ROOT / 'baseline.json').write_text(json.dumps({'ready': True, 'http_status': 200}) + '\n') expected = peer() before = counts() (ROOT / 'counts-before.json').write_text(json.dumps(before, sort_keys=True) + '\n') diff --git a/deployment/scripts/roll-to-format5-rehearsal/run.py b/deployment/scripts/roll-to-format5-rehearsal/run.py index c7798223d5..9815728368 100755 --- a/deployment/scripts/roll-to-format5-rehearsal/run.py +++ b/deployment/scripts/roll-to-format5-rehearsal/run.py @@ -46,15 +46,19 @@ def ignore_private(directory, names): def main(): ap = argparse.ArgumentParser(description=__doc__) - for name in ('old-archive', 'new-archive', 'old-checksums', 'new-checksums', + for name in ('new-archive', 'new-checksums', 'work-dir',): ap.add_argument('--' + name, required=True, type=Path) ap.add_argument('--store', required=True, action='append', type=Path) - ap.add_argument('--arch', choices=('arm64', 'amd64'), default='arm64') + ap.add_argument('--old-binary', required=True, action='append', type=Path) + ap.add_argument('--old-sha256', required=True, action='append') + ap.add_argument('--arch', choices=('arm64', 'amd64'), default='amd64') ap.add_argument('--case', choices=CASES, action='append') ap.add_argument('--conversion-minutes', type=int, default=90) ap.add_argument('--ready-seconds', type=int, default=300) args = ap.parse_args() + if len(args.old_binary) != len(args.store) or len(args.old_sha256) != len(args.store): + ap.error('each store requires its matching old binary and SHA-256') here = Path(__file__).resolve().parent work = args.work_dir.resolve() work.mkdir(parents=True, exist_ok=False) @@ -62,8 +66,13 @@ def main(): active = None image = 'sdn-format5-rehearsal:' + str(os.getpid()) try: - verified = [verify(args.old_archive, args.old_checksums), - verify(args.new_archive, args.new_checksums)] + verified = [verify(args.new_archive, args.new_checksums)] + for binary, expected in zip(args.old_binary, args.old_sha256): + with binary.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + if digest != expected: + raise ValueError('old binary checksum mismatch') + verified.append({'old_binary': binary.name, 'sha256': digest}) (work / 'archives.json').write_text(json.dumps(verified, indent=2) + '\n') context = work / 'image' context.mkdir() @@ -76,13 +85,17 @@ def main(): directory = work / ('store-' + str(number)) / case directory.mkdir(parents=True) shutil.copytree(source, directory / 'store', ignore=ignore_private) - unpack(args.old_archive, directory / 'base/sdn-bundle') + old_bundle = directory / 'base/sdn-bundle' + (old_bundle / 'bin').mkdir(parents=True) + old_binary = args.old_binary[number - 1] + shutil.copy2(old_binary, old_bundle / 'bin/spacedatanetwork') + shutil.copy2(old_binary.parent / 'manifest.json', old_bundle / 'manifest.json') unpack(args.new_archive, directory / 'base/staged') shutil.copy2(here.parent / 'roll-to-format5.sh', directory / 'roll.sh') shutil.copy2(here / 'case.py', directory / 'case.py') active = 'sdn-format5-' + str(os.getpid()) + '-' + str(number) + '-' + case with open(directory / 'container.log', 'w') as log: - subprocess.run(['docker', 'run', '-d', '--name', active, '--hostname=sdn-rehearsal', '--privileged', + subprocess.run(['docker', 'run', '-d', '-t', '--name', active, '--hostname=sdn-rehearsal', '--privileged', '--cgroupns=host', '--network=none', '--platform=linux/' + args.arch, '--cpus=2', '--memory=6g', '--tmpfs=/run', '--tmpfs=/run/lock', '--tmpfs=/tmp', '-v', '/sys/fs/cgroup:/sys/fs/cgroup:rw', From 831deea612db56c6f417c3b371daf714060ad33d Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 09:35:02 -0500 Subject: [PATCH 08/31] Exercise the disk gate on a bounded filesystem Create a sparse ext4 loop filesystem before the old node starts so the precheck observes insufficient real free space without a rollout override. Co-Authored-By: Codex --- .../scripts/roll-to-format5-rehearsal/README.md | 9 ++++++--- .../scripts/roll-to-format5-rehearsal/case.py | 17 +++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 6032de2f24..9b627ed0b4 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -59,11 +59,14 @@ silently create an empty store. | budget | One-minute conversion deadline, automatic recovery and identical fsql4 file hashes. A fast conversion that completes inside the budget fails this injection case. | | after-switch | A unit pre-start condition rejects the new fsql5 selection. The original bundle restarts after recovery removes that selection. | | wrong-peer | Wrong expected identity is refused before stopping, with unchanged PID, fsql4 hashes and bundle/store selection. | -| disk | Insufficient free bytes on the supplied filesystem are refused before stopping. This case is explicitly not run when space is sufficient. | +| disk | Insufficient free bytes on the supplied filesystem are refused before stopping. A bounded ext4 loop filesystem supplies the space constraint. | | script-term / script-hup | Signal to the exact roll-script MainPID during conversion invokes automatic recovery. | -The disk case does not fill a host disk or add a test-only script argument. -Supply a work directory on a genuinely constrained filesystem for that case. +The disk case creates a sparse ext4 image with capacity equal to the copied +store size plus 2 GiB, mounts it with a loop device, and copies the store into +that filesystem before startup. It checks the real free-space result against +the production formula. It does not fill a host disk or add a test-only script +argument. The image and original copy remain as evidence. Copied-store hashes are taken after the old node first becomes ready. The node can perform background maintenance; a hash mismatch is reported as failure, not waived. The public `/api/v1/stats` response supplies schema/count aggregates; diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 8aae0351ae..01e32b0ee4 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -6,6 +6,7 @@ from pathlib import Path import re import signal +import shutil import subprocess import sys import time @@ -66,6 +67,19 @@ def tree_size(path): def run(case, minutes, seconds): if Path('/proc/1/comm').read_text().strip() != 'systemd': raise RuntimeError('PID 1 is not systemd') + if case == 'disk': + # A real sparse loop filesystem bounds free space without filling a host disk. + source = ROOT / 'store-source' + (ROOT / 'store').rename(source) + (ROOT / 'store').mkdir() + image = ROOT / 'disk.img' + size = tree_size(source) + 2 * 1024**3 + command('truncate', '-s', str(size), str(image)) + command('mkfs.ext4', '-q', '-F', '-m', '0', str(image)) + command('mount', '-o', 'loop', str(image), str(ROOT / 'store')) + shutil.copytree(source, ROOT / 'store', dirs_exist_ok=True) + (ROOT / 'disk-filesystem.json').write_text(json.dumps({ + 'filesystem': 'ext4', 'capacity_bytes': size}) + '\n') (ROOT / 'home').mkdir() (ROOT / 'cache').mkdir() password = ROOT / 'test-password' @@ -141,8 +155,7 @@ def run(case, minutes, seconds): ''') command('systemctl', 'daemon-reload') if case == 'disk': - # Use a genuinely constrained filesystem supplied as the work directory. - # Never fill the host's disk to manufacture a failure. + # Verify that the actual filesystem fails the production space formula. vfs = os.statvfs(ROOT / 'store') available = vfs.f_bavail * vfs.f_frsize required = 3 * tree_size(ROOT / 'store/fsql4') + (ROOT / 'store/auxiliary.flatsqlmeta').stat().st_size + 1073741824 + 5000000000 From d7e4aeba592e6eeccfe081a27294aa4c148290ef Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 09:37:35 -0500 Subject: [PATCH 09/31] Read the rehearsal identity from the running node journal Use the current systemd invocation so standalone old binaries do not need an external wallet module for a separate identity command. Co-Authored-By: Codex --- deployment/scripts/roll-to-format5-rehearsal/case.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 01e32b0ee4..9fd410981a 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -57,7 +57,13 @@ def counts(): def peer(): - return command(str(BIN), 'show-identity', '-c', '/case/config.yaml') + invocation = command('systemctl', 'show', '-p', 'InvocationID', '--value', UNIT) + journal = command('journalctl', '--no-pager', '-o', 'cat', + '_SYSTEMD_INVOCATION_ID=' + invocation, '--grep=Peer ID: ') + matches = re.findall(r'Peer ID: ([a-zA-Z0-9]+)', journal) + if not matches: + raise RuntimeError('running node has no journal identity') + return matches[-1] def tree_size(path): From d4890f479600625a95f1b7e72afc57d036bc3c15 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 09:43:22 -0500 Subject: [PATCH 10/31] Configure isolated aggregate reads and reuse engine artifacts Expose stats on the loopback rehearsal listener without credentials and share only the compiled engine cache across fresh-copy cases. Co-Authored-By: Codex --- deployment/scripts/roll-to-format5-rehearsal/README.md | 8 ++++++-- deployment/scripts/roll-to-format5-rehearsal/case.py | 6 +++--- deployment/scripts/roll-to-format5-rehearsal/run.py | 4 ++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 9b627ed0b4..24d1ec970b 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -40,13 +40,17 @@ it creates. It retains case directories, logs, hashes, ledgers and results. The source stores are read only. Copies omit `keys`, `secrets` and `auth.db`. Every case starts its old node from a fresh copy, with a generated test identity, -a generated test password, a separate engine cache, a stable container hostname, +a generated test password, an engine cache separate from identity material, a stable container hostname, loopback listeners, disabled DHT and mDNS, no trusted peers, no host ports and `--network=none`. The stable hostname allows the release to create a durable test identity without Docker's ephemeral-hostname refusal. These releases replace an empty bootstrap list with public defaults. A synthetic peer identity at unused loopback port 9 suppresses that fallback without naming a real peer. -Docker's network namespace has no external interface. The new release's engine +Docker's network namespace has no external interface. The loopback admin +listener has `require_auth: false` to expose aggregate stats without credentials. +The running peer identity comes from the current systemd invocation journal. +`--cache-dir` reuses compiled engine artifacts across cases; identity material +is never placed in that directory. The new release's engine is prewarmed before stopping the old node. No runtime permission patch or fake systemctl is used. A failure to start the old release is recorded as a failed case and requires inspection; the harness does not replace the release or diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 9fd410981a..7b608b9035 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -87,12 +87,11 @@ def run(case, minutes, seconds): (ROOT / 'disk-filesystem.json').write_text(json.dumps({ 'filesystem': 'ext4', 'capacity_bytes': size}) + '\n') (ROOT / 'home').mkdir() - (ROOT / 'cache').mkdir() password = ROOT / 'test-password' password.write_text(os.urandom(32).hex()) password.chmod(0o600) os.environ['SDN_KEY_PASSWORD_FILE'] = str(password) - os.environ['XDG_CACHE_HOME'] = '/case/cache' + os.environ['XDG_CACHE_HOME'] = '/engine-cache' os.environ['HOME'] = '/case/home' # Empty bootstrap lists trigger built-in public defaults in these releases. # A synthetic identity at an unused loopback port prevents that fallback. @@ -125,6 +124,7 @@ def run(case, minutes, seconds): enabled: true listen_addr: 127.0.0.1:5001 tls_mode: disabled + require_auth: false setup: data_path: /case/store '''.replace('ISOLATED_PEER', isolated_peer)) @@ -134,7 +134,7 @@ def run(case, minutes, seconds): [Service] Type=simple User=root -Environment=HOME=/case/home XDG_CACHE_HOME=/case/cache SDN_KEY_PASSWORD_FILE=/case/test-password +Environment=HOME=/case/home XDG_CACHE_HOME=/engine-cache SDN_KEY_PASSWORD_FILE=/case/test-password ExecStart=/case/base/sdn-bundle/bin/spacedatanetwork daemon --config /case/config.yaml Restart=no TimeoutStopSec=75 diff --git a/deployment/scripts/roll-to-format5-rehearsal/run.py b/deployment/scripts/roll-to-format5-rehearsal/run.py index 9815728368..ca9a2b7f92 100755 --- a/deployment/scripts/roll-to-format5-rehearsal/run.py +++ b/deployment/scripts/roll-to-format5-rehearsal/run.py @@ -49,6 +49,7 @@ def main(): for name in ('new-archive', 'new-checksums', 'work-dir',): ap.add_argument('--' + name, required=True, type=Path) + ap.add_argument('--cache-dir', type=Path, help='shared engine cache; contains no identities') ap.add_argument('--store', required=True, action='append', type=Path) ap.add_argument('--old-binary', required=True, action='append', type=Path) ap.add_argument('--old-sha256', required=True, action='append') @@ -62,6 +63,8 @@ def main(): here = Path(__file__).resolve().parent work = args.work_dir.resolve() work.mkdir(parents=True, exist_ok=False) + cache = (args.cache_dir or (work / 'cache')).resolve() + cache.mkdir(parents=True, exist_ok=True) results = [] active = None image = 'sdn-format5-rehearsal:' + str(os.getpid()) @@ -99,6 +102,7 @@ def main(): '--cgroupns=host', '--network=none', '--platform=linux/' + args.arch, '--cpus=2', '--memory=6g', '--tmpfs=/run', '--tmpfs=/run/lock', '--tmpfs=/tmp', '-v', '/sys/fs/cgroup:/sys/fs/cgroup:rw', + '-v', str(cache) + ':/engine-cache', '-v', str(directory) + ':/case', image], stdout=log, stderr=subprocess.STDOUT, check=True) try: From 5539b3ea09133eec7f288a03a7eba737d807f4a4 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 09:50:36 -0500 Subject: [PATCH 11/31] Declare the isolated flow set and retain failed setup evidence Prewarm engine artifacts with no provider services configured. Permit a forward setup retry only before rollout, retaining previous attempt logs and verifying the old bundle before reuse. Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/README.md | 9 +++- .../scripts/roll-to-format5-rehearsal/case.py | 7 +++- .../scripts/roll-to-format5-rehearsal/run.py | 41 ++++++++++++++----- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 24d1ec970b..1814329942 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -34,7 +34,11 @@ retains the release archive's launcher and runtime layout. Old binary arguments and their digests follow the order of the store arguments. Changing `--new-archive` selects another release, with its corresponding checksum file supplied through `--new-checksums`. The output directory must not exist. -A selective repeat uses `--case` with one or more case names. Start Docker before +A selective repeat uses `--case` with one or more case names. +`--retry-before-roll --case forward` reuses a failed forward setup only when +no rollout ledger or format-5 selection exists and the old binary still matches +its verified digest. Previous results and logs move into an `attempt-*` directory. +The test identity remains in place. Other cases always use fresh copies. Start Docker before running and quit it after the run. The harness stops and removes only containers it creates. It retains case directories, logs, hashes, ledgers and results. @@ -48,7 +52,8 @@ replace an empty bootstrap list with public defaults. A synthetic peer identity at unused loopback port 9 suppresses that fallback without naming a real peer. Docker's network namespace has no external interface. The loopback admin listener has `require_auth: false` to expose aggregate stats without credentials. -The running peer identity comes from the current systemd invocation journal. +Provider flow services are explicitly empty, so prewarm checks only the +engine artifacts required by the isolated node. The running peer identity comes from the current systemd invocation journal. `--cache-dir` reuses compiled engine artifacts across cases; identity material is never placed in that directory. The new release's engine is prewarmed before stopping the old node. No runtime permission patch or fake diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 7b608b9035..683043f7ff 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -86,9 +86,10 @@ def run(case, minutes, seconds): shutil.copytree(source, ROOT / 'store', dirs_exist_ok=True) (ROOT / 'disk-filesystem.json').write_text(json.dumps({ 'filesystem': 'ext4', 'capacity_bytes': size}) + '\n') - (ROOT / 'home').mkdir() + (ROOT / 'home').mkdir(exist_ok=True) password = ROOT / 'test-password' - password.write_text(os.urandom(32).hex()) + if not password.exists(): + password.write_text(os.urandom(32).hex()) password.chmod(0o600) os.environ['SDN_KEY_PASSWORD_FILE'] = str(password) os.environ['XDG_CACHE_HOME'] = '/engine-cache' @@ -104,6 +105,8 @@ def run(case, minutes, seconds): isolated_peer = '1' + encoded # All keys belong to this isolated test. The container has no external network. (ROOT / 'config.yaml').write_text('''mode: full +flows: + services: [] network: listen: [/ip4/127.0.0.1/tcp/4001] bootstrap: [/ip4/127.0.0.1/tcp/9/p2p/ISOLATED_PEER] diff --git a/deployment/scripts/roll-to-format5-rehearsal/run.py b/deployment/scripts/roll-to-format5-rehearsal/run.py index ca9a2b7f92..8d4fc57a93 100755 --- a/deployment/scripts/roll-to-format5-rehearsal/run.py +++ b/deployment/scripts/roll-to-format5-rehearsal/run.py @@ -49,6 +49,8 @@ def main(): for name in ('new-archive', 'new-checksums', 'work-dir',): ap.add_argument('--' + name, required=True, type=Path) + ap.add_argument('--retry-before-roll', action='store_true', + help='retry a failed forward setup, retaining its original copied store') ap.add_argument('--cache-dir', type=Path, help='shared engine cache; contains no identities') ap.add_argument('--store', required=True, action='append', type=Path) ap.add_argument('--old-binary', required=True, action='append', type=Path) @@ -60,9 +62,11 @@ def main(): args = ap.parse_args() if len(args.old_binary) != len(args.store) or len(args.old_sha256) != len(args.store): ap.error('each store requires its matching old binary and SHA-256') + if args.retry_before_roll and args.case != ['forward']: + ap.error('before-roll retry requires --case forward') here = Path(__file__).resolve().parent work = args.work_dir.resolve() - work.mkdir(parents=True, exist_ok=False) + work.mkdir(parents=True, exist_ok=args.retry_before_roll) cache = (args.cache_dir or (work / 'cache')).resolve() cache.mkdir(parents=True, exist_ok=True) results = [] @@ -78,7 +82,7 @@ def main(): verified.append({'old_binary': binary.name, 'sha256': digest}) (work / 'archives.json').write_text(json.dumps(verified, indent=2) + '\n') context = work / 'image' - context.mkdir() + context.mkdir(exist_ok=args.retry_before_roll) shutil.copy2(here / 'Dockerfile', context / 'Dockerfile') with open(work / 'image.log', 'w') as log: subprocess.run(['docker', 'build', '--platform=linux/' + args.arch, @@ -86,14 +90,31 @@ def main(): for number, source in enumerate(args.store, 1): for case in args.case or CASES: directory = work / ('store-' + str(number)) / case - directory.mkdir(parents=True) - shutil.copytree(source, directory / 'store', ignore=ignore_private) - old_bundle = directory / 'base/sdn-bundle' - (old_bundle / 'bin').mkdir(parents=True) - old_binary = args.old_binary[number - 1] - shutil.copy2(old_binary, old_bundle / 'bin/spacedatanetwork') - shutil.copy2(old_binary.parent / 'manifest.json', old_bundle / 'manifest.json') - unpack(args.new_archive, directory / 'base/staged') + if args.retry_before_roll: + previous = json.loads((directory / 'result.json').read_text()) + if (previous['status'] != 'failed' or (directory / 'ledger.log').exists() + or (directory / 'store/fsql5').exists() + or not (directory / 'base/staged').is_dir()): + raise ValueError('retry requires a failed setup before any rollout') + with (directory / 'base/sdn-bundle/bin/spacedatanetwork').open('rb') as stream: + if hashlib.file_digest(stream, 'sha256').hexdigest() != args.old_sha256[number - 1]: + raise ValueError('retry old bundle differs from the verified input') + attempt = directory / ('attempt-' + str(os.getpid())) + attempt.mkdir() + for name in ('result.json', 'baseline.json', 'counts-before.json', + 'hashes-before.json', 'prewarm.log', 'driver.log', + 'driver.exit', 'journal.log', 'container.log'): + if (directory / name).exists(): + shutil.move(str(directory / name), attempt / name) + else: + directory.mkdir(parents=True) + shutil.copytree(source, directory / 'store', ignore=ignore_private) + old_bundle = directory / 'base/sdn-bundle' + (old_bundle / 'bin').mkdir(parents=True) + old_binary = args.old_binary[number - 1] + shutil.copy2(old_binary, old_bundle / 'bin/spacedatanetwork') + shutil.copy2(old_binary.parent / 'manifest.json', old_bundle / 'manifest.json') + unpack(args.new_archive, directory / 'base/staged') shutil.copy2(here.parent / 'roll-to-format5.sh', directory / 'roll.sh') shutil.copy2(here / 'case.py', directory / 'case.py') active = 'sdn-format5-' + str(os.getpid()) + '-' + str(number) + '-' + case From 621ae7d8520f1c4053f1c3d92d3bc32184d32a16 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 09:55:25 -0500 Subject: [PATCH 12/31] Retain independent recovery and repeat-run observations Record readiness, running identity and aggregate equality separately, and exercise repeat refusal after a completed recovery before reporting an aggregate mismatch. Keep the original preservation assertion unchanged. Co-Authored-By: Codex --- .../scripts/roll-to-format5-rehearsal/case.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 683043f7ff..fdb698b6d6 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -222,16 +222,25 @@ def run(case, minutes, seconds): raise RuntimeError('identity changed') after = counts() (ROOT / 'counts-after.json').write_text(json.dumps(after, sort_keys=True) + '\n') + (ROOT / 'observations.json').write_text(json.dumps({ + 'ready': True, 'running_peer_unchanged': True, 'counts_equal': before == after, + 'roll_exit': code, 'injected': injected}) + '\n') + if any(' ' + event + ' ' in ledger for event in + ('SUCCESS', 'ROLLED_BACK_BEFORE_SWITCH', 'ROLLED_BACK_AFTER_SWITCH')): + replay_pid = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) + with open(ROOT / 'second-run.log', 'w') as log: + second = subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode + (ROOT / 'second-run.exit').write_text(str(second) + '\n') + unchanged = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) == replay_pid + (ROOT / 'second-run.json').write_text(json.dumps({ + 'refused': second != 0, 'daemon_pid_unchanged': unchanged}) + '\n') + if second == 0 or not unchanged: + raise RuntimeError('second run was not refused without restarting the node') if before != after: raise RuntimeError('per-standard counts changed') if case == 'forward': if code != 0 or ' SUCCESS ' not in ledger: raise RuntimeError('forward conversion failed') - with open(ROOT / 'second-run.log', 'w') as log: - second = subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode - (ROOT / 'second-run.exit').write_text(str(second) + '\n') - if second == 0: - raise RuntimeError('second success run was not refused') elif case in ('wrong-peer', 'disk'): if code == 0 or ' STOPPED ' in ledger or ' PRECHECK_FAILED ' not in ledger: raise RuntimeError('precheck did not refuse before stop') @@ -253,11 +262,6 @@ def run(case, minutes, seconds): (ROOT / 'hashes-after.json').write_text(json.dumps(after_hashes, sort_keys=True) + '\n') if case != 'after-switch' and before_hashes != after_hashes: raise RuntimeError('format-4 file hashes changed') - with open(ROOT / 'second-run.log', 'w') as log: - second = subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode - (ROOT / 'second-run.exit').write_text(str(second) + '\n') - if second == 0: - raise RuntimeError('second rollback run was not refused') return {'status': 'passed', 'roll_exit': code, 'seconds': round(time.monotonic()-start, 2), 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, 'peak_conversion_rss_kib_sampled': peak_kib, From 67cf9a469ee22ce1e09c4bb0f3389499807da982 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 10:00:03 -0500 Subject: [PATCH 13/31] Preserve hash and conversion measurements on failed rehearsals Write independent byte-preservation and conversion observations before the strict aggregate comparison. Retain every assertion and its failure. Co-Authored-By: Codex --- .../scripts/roll-to-format5-rehearsal/case.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index fdb698b6d6..214e074890 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -184,6 +184,7 @@ def run(case, minutes, seconds): peak_kib = 0 migration_start = None migration_end = None + switched_bytes = None while proc.poll() is None: ledger = (ROOT / 'ledger.log').read_text() if (ROOT / 'ledger.log').exists() else '' match = re.search(r' CONVERTING .*pid=(\d+)', ledger) @@ -213,6 +214,7 @@ def run(case, minutes, seconds): injected = True if ' SWITCHED ' in ledger and migration_end is None: migration_end = time.monotonic() + switched_bytes = tree_size(ROOT / 'store/fsql5') time.sleep(0.25) code = proc.wait() (ROOT / 'roll.exit').write_text(str(code) + '\n') @@ -222,9 +224,16 @@ def run(case, minutes, seconds): raise RuntimeError('identity changed') after = counts() (ROOT / 'counts-after.json').write_text(json.dumps(after, sort_keys=True) + '\n') + after_hashes = None + if case in ('interrupted', 'budget', 'script-term', 'script-hup', 'wrong-peer', 'disk'): + after_hashes = hashes() + (ROOT / 'hashes-after.json').write_text(json.dumps(after_hashes, sort_keys=True) + '\n') (ROOT / 'observations.json').write_text(json.dumps({ 'ready': True, 'running_peer_unchanged': True, 'counts_equal': before == after, - 'roll_exit': code, 'injected': injected}) + '\n') + 'roll_exit': code, 'injected': injected, + 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, + 'peak_conversion_rss_kib_sampled': peak_kib, 'switched_fsql5_bytes': switched_bytes, + 'format4_hashes_equal': before_hashes == after_hashes if after_hashes is not None else None}) + '\n') if any(' ' + event + ' ' in ledger for event in ('SUCCESS', 'ROLLED_BACK_BEFORE_SWITCH', 'ROLLED_BACK_AFTER_SWITCH')): replay_pid = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) @@ -246,7 +255,7 @@ def run(case, minutes, seconds): raise RuntimeError('precheck did not refuse before stop') if command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) != baseline_pid: raise RuntimeError('precheck changed the running daemon') - if before_hashes != hashes(): + if before_hashes != after_hashes: raise RuntimeError('precheck changed format 4') if not (ROOT / 'base/staged').exists() or (ROOT / 'store/fsql5').exists(): raise RuntimeError('precheck changed bundle or store selection') @@ -258,8 +267,6 @@ def run(case, minutes, seconds): raise RuntimeError('failure was not injected during conversion') if case == 'budget' and 'reason=conversion_budget' not in ledger: raise RuntimeError('conversion did not reach its budget') - after_hashes = hashes() - (ROOT / 'hashes-after.json').write_text(json.dumps(after_hashes, sort_keys=True) + '\n') if case != 'after-switch' and before_hashes != after_hashes: raise RuntimeError('format-4 file hashes changed') return {'status': 'passed', 'roll_exit': code, 'seconds': round(time.monotonic()-start, 2), From b678a40d440a37ea7972aa67e10068ec19f75769 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 11:23:37 -0500 Subject: [PATCH 14/31] Document the installed-binary rehearsal findings Record successful old-node baselines, intermittent runtime-guard refusal, and observed post-switch recovery while retaining the failed preservation gate and limits of the evidence. Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/README.md | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 1814329942..2567d3d5fe 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -93,18 +93,37 @@ The harness and recovery script require execution on both source copies before use on a production host. An image-build failure or incomplete case matrix means the deployment gate has not passed. -## Release baseline result +## Rehearsal findings -The published v1.1.0 linux/arm64 archive, verified against its release checksum -file, fails to open both supplied format-4 store copies. Startup reports -`P4_E_FORMAT` and `flatsql_init returned status -5` before readiness. The same -archive reaches readiness with an empty store in the same systemd image. -That control creates `fsql4/STORE` with bytes 4 and 5 equal to `05 00`, while -both supplied stores contain `04 00`. The binary reports version 1.1.0. -No store marker is relabeled and no node code is changed. +The published v1.1.0 linux/arm64 archive fails to open the supplied format-4 +copies and creates format-5 marker bytes under `fsql4` with an empty store. +The two installed old binaries, verified against their supplied digests, each +reach readiness on their matching copied store in the amd64 rehearsal. -The old-node baseline gate has failed. Forward conversion, failure recovery, -repeat invocation and precheck cases have no end-to-end result on these copies. -The v1.1.5 archive checksum passes, but its conversion and recovery behavior -remain unverified. The script requires a passing baseline and complete case -matrix before deployment. +Both forward attempts fail before conversion because the verified v1.1.5 +linux/amd64 binary's runtime self-test reports +`stop-reaches-every-thread=false`. The command refuses with +`lacks the SDN runtime patches`. Other invocations report that flag true and +enter conversion, so the cause of the invocation-dependent refusal is not +established. The guard is not bypassed and node code is unchanged. + +The first input's post-switch case verifies 45 partitions and reaches SWITCHED +1,414 seconds after CONVERTING. At that point the retained format-4 tree, +including rollback material, contains 7,409,106,490 bytes and format 5 contains +11,421,381,390 bytes. The new unit's deliberate pre-start failure triggers +recovery to the old node, ready with the same running peer. A second invocation +is refused without restarting that node. These measurements describe emulated +Docker execution and are not a forecast for a deployment host. + +The strict aggregate and format-4 preservation checks fail. Aggregate and hash +changes also occur in precheck cases that reject before stopping the old node +or converting anything. The comparisons include active-daemon activity and do +not isolate a converter cause. The failed assertions remain in the harness. +Independent readiness, identity, hash, conversion and repeat-run observations +are written to `observations.json` and `second-run.json` before the aggregate +assertion, so a count failure does not erase those observations. + +The script is not cleared for deployment. Recovery ledger events alone do not +establish the required preservation proof or a successful new-node forward run. +Each retained case result and exit file determines whether the complete case +passed; an intended fault that was never injected is not a tested recovery leg. From 4f3ce04529c28cc5b601f27e2344a448fa5605db Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 12:10:57 -0500 Subject: [PATCH 15/31] Check the staged runtime before stopping the node Compare rehearsal records by content address and exact bytes, and hash format 4 before recovery restarts the original service. Reclaim completed case stores while retaining separate record evidence and logs. Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/README.md | 69 ++++-- .../scripts/roll-to-format5-rehearsal/case.py | 66 ++++-- .../roll-to-format5-rehearsal/records.py | 206 ++++++++++++++++++ .../scripts/roll-to-format5-rehearsal/run.py | 58 +++-- deployment/scripts/roll-to-format5.md | 25 ++- deployment/scripts/roll-to-format5.sh | 30 +++ 6 files changed, 385 insertions(+), 69 deletions(-) create mode 100644 deployment/scripts/roll-to-format5-rehearsal/records.py diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 2567d3d5fe..031ec6391b 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -2,7 +2,8 @@ This harness is an end-to-end test of `../roll-to-format5.sh`. It requires Docker with privileged containers, a cgroup mount usable by systemd, Python 3.12 -or later on the driver, and enough disk for independent copies of every case. +or later on the driver, and enough disk for a case store and record evidence. Completed case stores +and bundles are removed before the following case starts. The image uses Ubuntu 24.04 for the selected node architecture, amd64 by default. Systemd and its support libraries run on the build platform's native architecture; node binaries and target runtime libraries retain the selected architecture. @@ -24,23 +25,25 @@ python3 run.py \ --arch amd64 \ --store "$FIRST_STORE_COPY" --store "$SECOND_STORE_COPY" \ --work-dir "$NEW_REHEARSAL_DIRECTORY" \ + --record-evidence-dir "$PRIVATE_RECORD_EVIDENCE_DIRECTORY" \ --conversion-minutes 90 --ready-seconds 300 ``` Use the binary matching each store's installed build as OLD and published -v1.1.5 as NEW. Each old binary has its bundle's `manifest.json` beside it. +the selected release archive as NEW. Each old binary has its bundle's `manifest.json` beside it. The old bundle contains `bin/spacedatanetwork` and that manifest; the new bundle retains the release archive's launcher and runtime layout. Old binary arguments and their digests follow the order of the store arguments. Changing `--new-archive` selects another release, with its corresponding checksum file supplied through `--new-checksums`. The output directory must not exist. A selective repeat uses `--case` with one or more case names. -`--retry-before-roll --case forward` reuses a failed forward setup only when -no rollout ledger or format-5 selection exists and the old binary still matches -its verified digest. Previous results and logs move into an `attempt-*` directory. -The test identity remains in place. Other cases always use fresh copies. Start Docker before +Every repeat uses a fresh output directory, evidence directory and store copy. +Failed runtime checks remain failures; a repeat does not alter their results. +Start Docker before running and quit it after the run. The harness stops and removes only containers -it creates. It retains case directories, logs, hashes, ledgers and results. +it creates. It retains case logs, hashes, ledgers and results. Completed case +stores, backup directories, bundles and disk images are removed; record evidence +stays in the explicitly supplied private evidence directory. The source stores are read only. Copies omit `keys`, `secrets` and `auth.db`. Every case starts its old node from a fresh copy, with a generated test identity, @@ -63,11 +66,11 @@ silently create an empty store. | Case | Required evidence | | --- | --- | -| forward | Zero exit, SUCCESS, identical per-standard public stats, ready and unchanged identity. A second invocation is refused. | +| forward | Zero exit, SUCCESS, record-by-record preservation, ready and unchanged identity. A second invocation is refused. | | interrupted | SIGTERM to the recorded migration PID after the pending directory appears, automatic recovery, ready, unchanged identity and identical fsql4 file hashes. | | budget | One-minute conversion deadline, automatic recovery and identical fsql4 file hashes. A fast conversion that completes inside the budget fails this injection case. | | after-switch | A unit pre-start condition rejects the new fsql5 selection. The original bundle restarts after recovery removes that selection. | -| wrong-peer | Wrong expected identity is refused before stopping, with unchanged PID, fsql4 hashes and bundle/store selection. | +| wrong-peer | Wrong expected identity is refused before stopping, with unchanged PID, preserved records and bundle/store selection. | | disk | Insufficient free bytes on the supplied filesystem are refused before stopping. A bounded ext4 loop filesystem supplies the space constraint. | | script-term / script-hup | Signal to the exact roll-script MainPID during conversion invokes automatic recovery. | @@ -75,11 +78,34 @@ The disk case creates a sparse ext4 image with capacity equal to the copied store size plus 2 GiB, mounts it with a loop device, and copies the store into that filesystem before startup. It checks the real free-space result against the production formula. It does not fill a host disk or add a test-only script -argument. The image and original copy remain as evidence. -Copied-store hashes are taken after the old node first becomes ready. The node -can perform background maintenance; a hash mismatch is reported as failure, -not waived. The public `/api/v1/stats` response supplies schema/count aggregates; -an empty response or a changed count fails rather than claiming preservation. +argument. The image and original copy are removed after the case. + +Record preservation uses the node's anonymous `/api/v1/data/scan` endpoint. +The driver waits for startup connector persistence, then scans every standard +through the snapshot cursor. It checks the advertised total against every +record read. Before-state records stay in size-prefixed FlatBuffer streams in +`--record-evidence-dir`; an index contains SHA-256 content-address digests and +stream offsets. The after-state scan looks up every content address and +compares the complete bytes against that stream. Missing records, different +bytes, duplicate results and unexplained additions fail. Counts remain +observations rather than the preservation criterion. + +The only allowed additions are startup ICN records: each must name a connector +present before, have an update timestamp after the rollout began, and occur +once per connector. Their number must equal the current invocation's logged +startup persistence count. No additions are allowed when the old invocation +remains running. `ConnectorsHandler.persistLoop` calls `PersistConnectors` +after a 15-second boot delay; `BuildConnectorFrames` builds one record per +provider/source pair from stored summaries and the source ledger and assigns +a fresh `UPDATED_AT` timestamp. This accounts for startup records without +allowing arbitrary changes to existing records. + +Format-4 file hashes are taken at the script's explicit rehearsal barriers, +after the original service stops and after the bundle is restored but before +recovery starts that service. Only a newly added +`rollback-auxiliary.flatsqlmeta` is allowed; removed or changed files fail. +The conversion adds that snapshot immediately before activation. The barriers +are disabled unless the rehearsal explicitly supplies its pause directory. `results.json` summarizes case outcomes. Each case contains `driver.exit`, `roll.exit`, `roll.log`, `ledger.log`, aggregate `counts-*.json`, file hash lists, @@ -115,13 +141,14 @@ recovery to the old node, ready with the same running peer. A second invocation is refused without restarting that node. These measurements describe emulated Docker execution and are not a forecast for a deployment host. -The strict aggregate and format-4 preservation checks fail. Aggregate and hash -changes also occur in precheck cases that reject before stopping the old node -or converting anything. The comparisons include active-daemon activity and do -not isolate a converter cause. The failed assertions remain in the harness. -Independent readiness, identity, hash, conversion and repeat-run observations -are written to `observations.json` and `second-run.json` before the aggregate -assertion, so a count failure does not erase those observations. +The preceding rehearsal's aggregate comparisons differed only in ICN counts: +33 additional records on the first input and 11 on the second. Startup journal +lines report exactly those persistence counts, including the precheck controls +that never stop or convert the store. Active-daemon file hashes cannot isolate +conversion changes. The record comparison and stopped-state barriers replace +those measurements; they require a completed rehearsal before they establish +preservation. The earlier failing results remain evidence of those earlier +measurements. The script is not cleared for deployment. Recovery ledger events alone do not establish the required preservation proof or a successful new-node forward run. diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 214e074890..3daec978a6 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -12,6 +12,8 @@ import time import urllib.request +import records + ROOT = Path('/case') UNIT = 'rehearsal-node.service' READY = 'http://127.0.0.1:5001/api/v1/ready' @@ -66,6 +68,19 @@ def peer(): return matches[-1] +def boot_observation(): + invocation = command('systemctl', 'show', '-p', 'InvocationID', '--value', UNIT) + end = time.monotonic() + 180 + while time.monotonic() < end: + journal = command('journalctl', '--no-pager', '-o', 'cat', + '_SYSTEMD_INVOCATION_ID=' + invocation) + matches = re.findall(r'Connector lane: persisted (\d+) \$ICN row', journal) + if matches: + return invocation, sum(map(int, matches)) + time.sleep(1) + raise RuntimeError('connector startup persistence did not finish') + + def tree_size(path): return sum(p.stat().st_size for p in path.rglob('*') if p.is_file()) if path.exists() else 0 @@ -148,10 +163,14 @@ def run(case, minutes, seconds): command('systemctl', 'is-active', '--quiet', UNIT) (ROOT / 'baseline.json').write_text(json.dumps({'ready': True, 'http_status': 200}) + '\n') expected = peer() + before_invocation, _ = boot_observation() before = counts() (ROOT / 'counts-before.json').write_text(json.dumps(before, sort_keys=True) + '\n') - before_hashes = hashes() - (ROOT / 'hashes-before.json').write_text(json.dumps(before_hashes, sort_keys=True) + '\n') + records.snapshot('/evidence/records-before', before) + before_hashes = None + stopped_hash_result = None + pause = ROOT / 'pause' + pause.mkdir() # Populate each release's engine cache before the outage. with open(ROOT / 'prewarm.log', 'w') as log: subprocess.run(['/case/base/staged/bin/spacedatanetwork', 'prewarm-aot', '--config', @@ -178,7 +197,9 @@ def run(case, minutes, seconds): str(seconds), '/case/home'] baseline_pid = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) start = time.monotonic() - proc = subprocess.Popen(['systemd-run', '--unit=rehearsal-roll', '--collect', '--wait', '--pipe', *args], + boot_ms = int(time.time() * 1000) + proc = subprocess.Popen(['systemd-run', '--unit=rehearsal-roll', '--collect', '--wait', '--pipe', + '--setenv=SDN_ROLL_REHEARSAL_PAUSE_DIR=/case/pause', *args], stdout=open(ROOT / 'roll.log', 'w'), stderr=subprocess.STDOUT) injected = False peak_kib = 0 @@ -186,6 +207,25 @@ def run(case, minutes, seconds): migration_end = None switched_bytes = None while proc.poll() is None: + for point in ('stopped', 'restored'): + if (pause / (point + '.waiting')).exists() and not (pause / (point + '.continue')).exists(): + if command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) != '0': + raise RuntimeError('hash barrier has a running node') + current = hashes() + (ROOT / ('hashes-' + point + '.json')).write_text(json.dumps(current, sort_keys=True) + '\n') + if point == 'stopped': + before_hashes = current + elif case != 'after-switch': + added = set(current) - set(before_hashes) + missing = set(before_hashes) - set(current) + changed = [key for key in set(before_hashes) & set(current) + if before_hashes[key] != current[key]] + stopped_hash_result = {'passed': not missing and not changed and + added <= {'rollback-auxiliary.flatsqlmeta'}, + 'missing': len(missing), 'changed': len(changed), + 'added': sorted(added)} + (ROOT / 'stopped-hash-comparison.json').write_text(json.dumps(stopped_hash_result) + '\n') + (pause / (point + '.continue')).touch() ledger = (ROOT / 'ledger.log').read_text() if (ROOT / 'ledger.log').exists() else '' match = re.search(r' CONVERTING .*pid=(\d+)', ledger) if match: @@ -222,18 +262,18 @@ def run(case, minutes, seconds): wait_ready(seconds) if peer() != expected: raise RuntimeError('identity changed') + after_invocation, boot_additions = boot_observation() after = counts() (ROOT / 'counts-after.json').write_text(json.dumps(after, sort_keys=True) + '\n') - after_hashes = None - if case in ('interrupted', 'budget', 'script-term', 'script-hup', 'wrong-peer', 'disk'): - after_hashes = hashes() - (ROOT / 'hashes-after.json').write_text(json.dumps(after_hashes, sort_keys=True) + '\n') + preservation = records.compare('/evidence/records-before', after, boot_ms, + boot_additions if after_invocation != before_invocation else 0) + (ROOT / 'record-preservation.json').write_text(json.dumps(preservation, indent=2) + '\n') (ROOT / 'observations.json').write_text(json.dumps({ 'ready': True, 'running_peer_unchanged': True, 'counts_equal': before == after, 'roll_exit': code, 'injected': injected, 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, 'peak_conversion_rss_kib_sampled': peak_kib, 'switched_fsql5_bytes': switched_bytes, - 'format4_hashes_equal': before_hashes == after_hashes if after_hashes is not None else None}) + '\n') + 'stopped_format4_hashes': stopped_hash_result, 'records_preserved': preservation['passed']}) + '\n') if any(' ' + event + ' ' in ledger for event in ('SUCCESS', 'ROLLED_BACK_BEFORE_SWITCH', 'ROLLED_BACK_AFTER_SWITCH')): replay_pid = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) @@ -245,8 +285,8 @@ def run(case, minutes, seconds): 'refused': second != 0, 'daemon_pid_unchanged': unchanged}) + '\n') if second == 0 or not unchanged: raise RuntimeError('second run was not refused without restarting the node') - if before != after: - raise RuntimeError('per-standard counts changed') + if not preservation['passed']: + raise RuntimeError('record preservation failed') if case == 'forward': if code != 0 or ' SUCCESS ' not in ledger: raise RuntimeError('forward conversion failed') @@ -255,8 +295,6 @@ def run(case, minutes, seconds): raise RuntimeError('precheck did not refuse before stop') if command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) != baseline_pid: raise RuntimeError('precheck changed the running daemon') - if before_hashes != after_hashes: - raise RuntimeError('precheck changed format 4') if not (ROOT / 'base/staged').exists() or (ROOT / 'store/fsql5').exists(): raise RuntimeError('precheck changed bundle or store selection') else: @@ -267,8 +305,8 @@ def run(case, minutes, seconds): raise RuntimeError('failure was not injected during conversion') if case == 'budget' and 'reason=conversion_budget' not in ledger: raise RuntimeError('conversion did not reach its budget') - if case != 'after-switch' and before_hashes != after_hashes: - raise RuntimeError('format-4 file hashes changed') + if case != 'after-switch' and not (stopped_hash_result or {}).get('passed'): + raise RuntimeError('stopped format-4 file hashes changed or were not measured') return {'status': 'passed', 'roll_exit': code, 'seconds': round(time.monotonic()-start, 2), 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, 'peak_conversion_rss_kib_sampled': peak_kib, diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py new file mode 100644 index 0000000000..3c04820c67 --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -0,0 +1,206 @@ +"""Compare every scanned record's content address and exact stored bytes. + +Raw evidence stays in size-prefixed FlatBuffer streams. The SQLite index holds +only digests and stream offsets. Public results contain per-standard totals. +""" +import hashlib +import json +from pathlib import Path +import sqlite3 +import struct +import urllib.request + +API = 'http://127.0.0.1:5001/api/v1/data/scan' +MEDIA = 'application/vnd.sdn.flatbuffers.stream' + + +def scalar(data, offset, fmt): + return struct.unpack_from('<' + fmt, data, offset)[0] + + +def field(data, slot): + table = scalar(data, 0, 'I') + vtable = table - scalar(data, table, 'i') + entry = 4 + slot * 2 + if entry >= scalar(data, vtable, 'H'): + return 0 + offset = scalar(data, vtable + entry, 'H') + return table + offset if offset else 0 + + +def number(data, slot, fmt): + offset = field(data, slot) + return scalar(data, offset, fmt) if offset else 0 + + +def string(data, slot): + offset = field(data, slot) + if not offset: + return b'' + start = offset + scalar(data, offset, 'I') + size = scalar(data, start, 'I') + return data[start + 4:start + 4 + size] + + +def request_frame(schema, cursor): + # QRP slots: KIND=0 (Request, the default), SCHEMA_NAME=5, + # LIMIT=20, CURSOR=21. A forward-built FlatBuffer needs no dependency. + vtable = 8 + table = 56 + data = bytearray(72) + struct.pack_into(' "$pause_dir/$point.waiting" || return 1 + log REHEARSAL_PAUSED "point=$point" || return 1 + while [[ ! -f $pause_dir/$point.continue ]]; do + (( SECONDS < end )) || return 1 + sleep 1 + done +} + # Parse systemctl's quoted words without evaluating shell code or printing values. # Unsupported escape spellings fail closed before the unit stops. unit_env=() @@ -162,6 +176,8 @@ rollback() { if exists "$live"; then mv -T -- "$live" "$failed" || return 1; fi mv -T -- "$old" "$live" || return 1 fi + stage=REHEARSAL_RESTORED + rehearsal_pause restored || return 1 stage=START_OLD systemctl start --no-block "$unit" || return 1 wait_ready || return 1 @@ -194,6 +210,12 @@ trap 'reason=SIGHUP; exit 129' HUP trap 'reason=SIGINT; exit 130' INT log PRECHECK 'begin' +if [[ -n $pause_dir ]]; then + [[ $pause_dir == /* && -d $pause_dir && ! -L $pause_dir ]] || fail rehearsal_pause_directory + for point in stopped restored; do + ! exists "$pause_dir/$point.waiting" && ! exists "$pause_dir/$point.continue" || fail rehearsal_pause_stale + done +fi [[ $(systemctl show -p LoadState --value "$unit") == loaded ]] || fail unit_missing systemctl is-active --quiet "$unit" || fail unit_inactive user=$(systemctl show -p User --value "$unit") @@ -224,11 +246,19 @@ free=$(df -B1 --output=avail -- "$root" | tail -n 1); free=${free// /} need=$((3 * bytes + aux + 1073741824 + 5000000000)) log DISK "required_bytes=$need available_bytes=$free" (( free >= need )) || fail disk_gate +log RUNTIME_CHECK 'begin' +if ! "${node_command[@]}" "$staged/bin/spacedatanetwork" substrate-selftest --require-patched; then + log RUNTIME_CHECK_FAILED + fail staged_runtime +fi +log RUNTIME_CHECK_OK log PRECHECK_OK stage=STOP stopped=1 stop_unit || fail stop_timeout log STOPPED +stage=REHEARSAL_STOPPED +rehearsal_pause stopped || fail rehearsal_pause_timeout stage=SWAP mv -T -- "$live" "$old" mv -T -- "$staged" "$live" From 11226af3c7c0f66d1c949ae691553a5dac214148 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 12:18:36 -0500 Subject: [PATCH 16/31] Retry transient runtime refusals within the conversion budget Record each runtime attempt and retain conversion output. Rehearse automatic service restarts with the deployment restart policy. Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/README.md | 11 +-- .../scripts/roll-to-format5-rehearsal/case.py | 8 +- .../scripts/roll-to-format5-rehearsal/run.py | 2 + deployment/scripts/roll-to-format5.md | 35 ++++++-- deployment/scripts/roll-to-format5.sh | 79 ++++++++++++++----- 5 files changed, 99 insertions(+), 36 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 031ec6391b..7306cb03f2 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -29,8 +29,8 @@ python3 run.py \ --conversion-minutes 90 --ready-seconds 300 ``` -Use the binary matching each store's installed build as OLD and published -the selected release archive as NEW. Each old binary has its bundle's `manifest.json` beside it. +Use the binary matching each store's installed build as OLD and the published +selected release archive as NEW. Each old binary has its bundle's `manifest.json` beside it. The old bundle contains `bin/spacedatanetwork` and that manifest; the new bundle retains the release archive's launcher and runtime layout. Old binary arguments and their digests follow the order of the store arguments. @@ -38,7 +38,8 @@ Changing `--new-archive` selects another release, with its corresponding checksu file supplied through `--new-checksums`. The output directory must not exist. A selective repeat uses `--case` with one or more case names. Every repeat uses a fresh output directory, evidence directory and store copy. -Failed runtime checks remain failures; a repeat does not alter their results. +Runtime prechecks and early migration runtime refusals use the script's bounded +retry policy. Exhausted retries remain failures; a repeat does not alter results. Start Docker before running and quit it after the run. The harness stops and removes only containers it creates. It retains case logs, hashes, ledgers and results. Completed case @@ -48,7 +49,7 @@ stays in the explicitly supplied private evidence directory. The source stores are read only. Copies omit `keys`, `secrets` and `auth.db`. Every case starts its old node from a fresh copy, with a generated test identity, a generated test password, an engine cache separate from identity material, a stable container hostname, -loopback listeners, disabled DHT and mDNS, no trusted peers, no host ports and +`Restart=always` with `RestartSec=15`, loopback listeners, disabled DHT and mDNS, no trusted peers, no host ports and `--network=none`. The stable hostname allows the release to create a durable test identity without Docker's ephemeral-hostname refusal. These releases replace an empty bootstrap list with public defaults. A synthetic peer identity @@ -108,7 +109,7 @@ The conversion adds that snapshot immediately before activation. The barriers are disabled unless the rehearsal explicitly supplies its pause directory. `results.json` summarizes case outcomes. Each case contains `driver.exit`, -`roll.exit`, `roll.log`, `ledger.log`, aggregate `counts-*.json`, file hash lists, +`roll.exit`, `roll.log`, conversion attempt logs, `ledger.log`, aggregate `counts-*.json`, file hash lists, `journal.log` and `result.json`. Peak migration RSS is sampled from `/proc`; conversion duration is sampled between CONVERTING and SWITCHED ledger events. All times describe Docker on the test machine. Archive digests are recorded in diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 3daec978a6..9f6cdbfc30 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -154,7 +154,8 @@ def run(case, minutes, seconds): User=root Environment=HOME=/case/home XDG_CACHE_HOME=/engine-cache SDN_KEY_PASSWORD_FILE=/case/test-password ExecStart=/case/base/sdn-bundle/bin/spacedatanetwork daemon --config /case/config.yaml -Restart=no +Restart=always +RestartSec=15 TimeoutStopSec=75 ''') command('systemctl', 'daemon-reload') @@ -227,9 +228,10 @@ def run(case, minutes, seconds): (ROOT / 'stopped-hash-comparison.json').write_text(json.dumps(stopped_hash_result) + '\n') (pause / (point + '.continue')).touch() ledger = (ROOT / 'ledger.log').read_text() if (ROOT / 'ledger.log').exists() else '' - match = re.search(r' CONVERTING .*pid=(\d+)', ledger) + matches = re.findall(r' CONVERTING .*pid=(\d+)', ledger) + match = matches[-1] if matches else None if match: - pid = int(match[1]) + pid = int(match) migration_start = migration_start or time.monotonic() stat = Path('/proc') / str(pid) / 'status' if stat.exists(): diff --git a/deployment/scripts/roll-to-format5-rehearsal/run.py b/deployment/scripts/roll-to-format5-rehearsal/run.py index ab0b2a4462..03df9d82b3 100755 --- a/deployment/scripts/roll-to-format5-rehearsal/run.py +++ b/deployment/scripts/roll-to-format5-rehearsal/run.py @@ -134,6 +134,8 @@ def main(): if (directory / name).exists(): shutil.rmtree(directory / name) for backup in directory.glob('store.roll-backup-*'): + for log in backup.glob('conversion-attempt-*.log'): + shutil.copy2(log, directory / log.name) shutil.rmtree(backup) (directory / 'disk.img').unlink(missing_ok=True) (work / 'results.json').write_text(json.dumps(results, indent=2) + '\n') diff --git a/deployment/scripts/roll-to-format5.md b/deployment/scripts/roll-to-format5.md index c5c16ca907..f25f03784d 100644 --- a/deployment/scripts/roll-to-format5.md +++ b/deployment/scripts/roll-to-format5.md @@ -68,7 +68,8 @@ journalctl -u "$ROLL_UNIT" -f The first budget is a positive whole number of minutes; the second is a positive whole number of seconds. The readiness budget applies separately to the new node and to recovery. Stop waits are bounded at 90 seconds. Conversion is -supervised as a child of the script; its output remains attached to the script. +supervised as a child of the script. Each attempt writes a log in the side-file +backup directory; CONVERTING names it. Completed attempt output is also printed. At the conversion deadline the script kills and waits for that exact child before inspecting the switch. It does not leave an independent migration job. The conversion command includes `--store STORE_ROOT` as well as `--config` to @@ -92,15 +93,17 @@ Every event is appended as one UTC line and also written to standard output: | --- | --- | | PRECHECK | Read-only checks started, apart from the ledger and the identity command's permission enforcement. | | DISK | Required and available byte counts. | -| RUNTIME_CHECK | The staged binary runs `substrate-selftest --require-patched` with the service environment. | +| RUNTIME_CHECK | `attempt=N`: the staged binary runs `substrate-selftest --require-patched` with the service environment. | | RUNTIME_CHECK_OK | The staged runtime passed all required checks. | -| RUNTIME_CHECK_FAILED | The staged runtime refused; the service remains running. | +| RUNTIME_CHECK_REFUSED | This precheck attempt failed. | +| RUNTIME_CHECK_FAILED | All five precheck attempts failed; the service remains running. | | PRECHECK_OK | Environment, identity, path, space and runtime checks passed. | | REHEARSAL_PAUSED | An explicitly enabled rehearsal barrier awaits acknowledgment. | | STOPPED | MainPID is zero and the unit is inactive or failed. | | SWAPPED | The old bundle is retained and the staged bundle occupies the live name. | | BACKED_UP | Stopped-state side files are copied beside the store. | | CONVERTING | Exact conversion PID and time budget. | +| CONVERSION_RETRY | An early runtime-patches refusal left no format-5 content; another attempt follows. | | SWITCHED | Format bytes are `05 00`; fsql4 and its auxiliary snapshot exist. | | SUCCESS | The new service answers ready and its invocation journal has the expected peer ID. | | FAILED | A checked step failed; recovery follows if stopping began. | @@ -169,8 +172,24 @@ barrier files are refused before the outage. Leave this variable unset in production; the default has no pause. The directory belongs to the rehearsal and contains no store or identity files. -The precheck measures the staged process. The migration process repeats its -own runtime guard, so a later refusal still invokes recovery. The self-test's -reported AOT speed ratio is diagnostic; the runtime's pass predicate checks -thread interruption, atomic operations and native execution. No threshold or -runtime check is weakened by this script. +The staged runtime precheck makes at most five attempts, ten seconds apart, +passing on the first success. Every attempt appears in the ledger. All five +failures refuse the rollout before stopping the service. + +The migration repeats its own runtime guard. An exit within the first 30 seconds +with the runtime-patches refusal is retried only when no format-5 candidate +content existed before or after that attempt and no fsql5 selection exists. +There are at most five conversion attempts, ten seconds apart, within one +conversion budget. Every other failure invokes recovery. No runtime check or +threshold is weakened. + +The deployment service uses `Restart=always` and `RestartSec=15`. A daemon +runtime refusal therefore schedules another start after 15 seconds, subject to +systemd's start limits. The readiness loop keeps polling across failed starts +and restarts for the entire ready budget; it does not stop at the first failed +invocation. Check the installed service's restart policy and start limits before +use. The isolated rehearsal uses the same restart settings. + +The self-test gives interpreter spinners two seconds to start and one executor +stop 300 milliseconds to reach every spinner. Its reported AOT speed ratio is +diagnostic and is not the runtime patch pass predicate. diff --git a/deployment/scripts/roll-to-format5.sh b/deployment/scripts/roll-to-format5.sh index 36033fe7af..cb93b5d93a 100755 --- a/deployment/scripts/roll-to-format5.sh +++ b/deployment/scripts/roll-to-format5.sh @@ -246,12 +246,21 @@ free=$(df -B1 --output=avail -- "$root" | tail -n 1); free=${free// /} need=$((3 * bytes + aux + 1073741824 + 5000000000)) log DISK "required_bytes=$need available_bytes=$free" (( free >= need )) || fail disk_gate -log RUNTIME_CHECK 'begin' -if ! "${node_command[@]}" "$staged/bin/spacedatanetwork" substrate-selftest --require-patched; then - log RUNTIME_CHECK_FAILED +runtime_ok=0 +for ((attempt=1; attempt<=5; attempt++)); do + log RUNTIME_CHECK "attempt=$attempt" + if "${node_command[@]}" "$staged/bin/spacedatanetwork" substrate-selftest --require-patched; then + runtime_ok=1 + log RUNTIME_CHECK_OK "attempt=$attempt" + break + fi + log RUNTIME_CHECK_REFUSED "attempt=$attempt" + (( attempt == 5 )) || sleep 10 +done +if (( ! runtime_ok )); then + log RUNTIME_CHECK_FAILED 'attempts=5' fail staged_runtime fi -log RUNTIME_CHECK_OK log PRECHECK_OK stage=STOP stopped=1 @@ -270,25 +279,55 @@ for name in "${side_files[@]}"; do done log BACKED_UP stage=CONVERT -# A child lets the shell handle signals while it supervises the foreground task. -signal_pending= -trap 'signal_pending=SIGTERM' TERM -trap 'signal_pending=SIGHUP' HUP -trap 'signal_pending=SIGINT' INT -"${node_command[@]}" "$live/bin/spacedatanetwork" store-migrate --to 5 --config "$config" --store "$root" & -conversion_pid=$! -trap 'reason=SIGTERM; exit 143' TERM -trap 'reason=SIGHUP; exit 129' HUP -trap 'reason=SIGINT; exit 130' INT -if [[ -n $signal_pending ]]; then reason=$signal_pending; exit 1; fi -log CONVERTING "pid=$conversion_pid budget_minutes=$minutes" +# Retry only an early runtime refusal with no candidate content. +format5_empty() ( + shopt -s nullglob dotglob + ! exists "$root/fsql5" || exit 1 + entries=("$root/.store-format5-pending"/*) + (( ${#entries[@]} == 0 )) +) deadline=$((SECONDS + minutes * 60)) -while kill -0 "$conversion_pid" 2>/dev/null; do +for ((attempt=1; attempt<=5; attempt++)); do (( SECONDS < deadline )) || fail conversion_budget - sleep 1 + empty_before=0 + format5_empty && empty_before=1 + attempt_log=$backup/conversion-attempt-$attempt.log + attempt_start=$SECONDS + # A child lets the shell handle signals while it supervises the task. + signal_pending= + trap 'signal_pending=SIGTERM' TERM + trap 'signal_pending=SIGHUP' HUP + trap 'signal_pending=SIGINT' INT + "${node_command[@]}" "$live/bin/spacedatanetwork" store-migrate --to 5 --config "$config" --store "$root" >"$attempt_log" 2>&1 & + conversion_pid=$! + trap 'reason=SIGTERM; exit 143' TERM + trap 'reason=SIGHUP; exit 129' HUP + trap 'reason=SIGINT; exit 130' INT + if [[ -n $signal_pending ]]; then reason=$signal_pending; exit 1; fi + log CONVERTING "pid=$conversion_pid budget_minutes=$minutes attempt=$attempt output=$attempt_log" + while kill -0 "$conversion_pid" 2>/dev/null; do + (( SECONDS < deadline )) || fail conversion_budget + sleep 1 + done + status=0 + wait "$conversion_pid" || status=$? + conversion_pid= + elapsed=$((SECONDS - attempt_start)) + cat -- "$attempt_log" + (( status != 0 )) || break + refused=0 + while IFS= read -r line; do + [[ $line != *'store-migrate --to 5 cannot run: the linked WasmEdge '*'lacks the SDN runtime patches'* ]] || refused=1 + done < "$attempt_log" + if (( attempt == 5 || ! refused || ! empty_before || elapsed >= 30 )) || ! format5_empty; then + fail conversion_exit + fi + log CONVERSION_RETRY "attempt=$attempt exit=$status elapsed_seconds=$elapsed reason=runtime_refused" + for ((delay=0; delay<10; delay++)); do + (( SECONDS < deadline )) || fail conversion_budget + sleep 1 + done done -wait "$conversion_pid" || fail conversion_exit -conversion_pid= stage=CHECK_SWITCH marker=$(od -An -tx1 -j4 -N2 -- "$root/fsql5/STORE"); marker=${marker// /} [[ $marker == 0500 && -d $root/fsql4 && -f $root/fsql4/rollback-auxiliary.flatsqlmeta ]] || fail invalid_switch From 4b7b8cdbce399595aba8e27c35817bd81fde529b Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 12:38:11 -0500 Subject: [PATCH 17/31] Compare complete record scans independently of source counts Stats counts sum source associations. Validate each scan against its own total and retain both measurements beside the exact byte comparison. Co-Authored-By: Codex --- deployment/scripts/roll-to-format5-rehearsal/README.md | 6 ++++-- deployment/scripts/roll-to-format5-rehearsal/records.py | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 7306cb03f2..6ba86ea0cd 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -83,8 +83,10 @@ argument. The image and original copy are removed after the case. Record preservation uses the node's anonymous `/api/v1/data/scan` endpoint. The driver waits for startup connector persistence, then scans every standard -through the snapshot cursor. It checks the advertised total against every -record read. Before-state records stay in size-prefixed FlatBuffer streams in +through the snapshot cursor. It checks the scan's advertised total against every +record read. Stats counts sum source associations and can exceed unique record +counts; both figures are retained in `scan-versus-stats.json`. Their equality +is not a record-preservation condition. Before-state records stay in size-prefixed FlatBuffer streams in `--record-evidence-dir`; an index contains SHA-256 content-address digests and stream offsets. The after-state scan looks up every content address and compares the complete bytes against that stream. Missing records, different diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py index 3c04820c67..84d6a37558 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/records.py +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -129,6 +129,7 @@ def snapshot(directory, schemas): directory = Path(directory) directory.mkdir(parents=True, exist_ok=False) results = {} + stats_comparison = {} for schema in sorted(schemas): if not schema.replace('_', '').replace('.fbs', '').isalnum() or not schema.endswith('.fbs'): raise ValueError('unexpected schema spelling') @@ -151,8 +152,11 @@ def snapshot(directory, schemas): db.commit() db.commit() db.close() - if count != schemas[schema]: - raise RuntimeError('baseline scan count differs from stats for ' + schema) + # Stats sums source associations; a raw scan enumerates content addresses. + # The scan validates its own advertised total against all frames read. + stats_comparison[schema] = {'stats_source_count': schemas[schema], + 'scanned_records': count} + (directory / 'scan-versus-stats.json').write_text(json.dumps(stats_comparison, sort_keys=True) + '\n') results[schema] = count (directory / 'summary.json').write_text(json.dumps(results, sort_keys=True) + '\n') return results From d75cd9a7dead93b3fbccb92c4d07d4f2501d4712 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 12:54:34 -0500 Subject: [PATCH 18/31] Keep mutable record indexes in the container temporary filesystem Retain completed baseline indexes with the evidence and compare against temporary copies. The record completeness and byte checks are unchanged. Co-Authored-By: Codex --- .../scripts/roll-to-format5-rehearsal/README.md | 4 +++- .../scripts/roll-to-format5-rehearsal/records.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 6ba86ea0cd..4b5bfa24df 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -88,7 +88,9 @@ record read. Stats counts sum source associations and can exceed unique record counts; both figures are retained in `scan-versus-stats.json`. Their equality is not a record-preservation condition. Before-state records stay in size-prefixed FlatBuffer streams in `--record-evidence-dir`; an index contains SHA-256 content-address digests and -stream offsets. The after-state scan looks up every content address and +stream offsets. Working indexes live in the container temporary filesystem; +completed baseline indexes are copied into the evidence directory. Comparison +uses a temporary copy so the baseline evidence stays unchanged. The after-state scan looks up every content address and compares the complete bytes against that stream. Missing records, different bytes, duplicate results and unexplained additions fail. Counts remain observations rather than the preservation criterion. diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py index 84d6a37558..4e0465618a 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/records.py +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -7,6 +7,8 @@ import json from pathlib import Path import sqlite3 +import shutil +import tempfile import struct import urllib.request @@ -133,7 +135,9 @@ def snapshot(directory, schemas): for schema in sorted(schemas): if not schema.replace('_', '').replace('.fbs', '').isalnum() or not schema.endswith('.fbs'): raise ValueError('unexpected schema spelling') - db = sqlite3.connect(directory / (schema + '.sqlite')) + temporary = tempfile.TemporaryDirectory(prefix='record-snapshot-') + index = Path(temporary.name) / 'index.sqlite' + db = sqlite3.connect(index) db.execute('CREATE TABLE records (cid BLOB PRIMARY KEY, offset INTEGER, size INTEGER, seen INTEGER DEFAULT 0) WITHOUT ROWID') db.execute('CREATE TABLE connectors (id BLOB PRIMARY KEY) WITHOUT ROWID') count = 0 @@ -152,6 +156,8 @@ def snapshot(directory, schemas): db.commit() db.commit() db.close() + shutil.copy2(index, directory / (schema + '.sqlite')) + temporary.cleanup() # Stats sums source associations; a raw scan enumerates content addresses. # The scan validates its own advertised total against all frames read. stats_comparison[schema] = {'stats_source_count': schemas[schema], @@ -170,7 +176,10 @@ def compare(directory, schemas, boot_ms, expected_boot_additions): if schema not in before: results[schema] = {'unexpected_standard': True} continue - db = sqlite3.connect(directory / (schema + '.sqlite')) + temporary = tempfile.TemporaryDirectory(prefix='record-compare-') + index = Path(temporary.name) / 'index.sqlite' + shutil.copy2(directory / (schema + '.sqlite'), index) + db = sqlite3.connect(index) added = changed = unexplained = matched = 0 added_connectors = set() with (directory / (schema + '.sds')).open('rb') as stream: @@ -199,6 +208,7 @@ def compare(directory, schemas, boot_ms, expected_boot_additions): missing = db.execute('SELECT count(*) FROM records WHERE seen=0').fetchone()[0] db.commit() db.close() + temporary.cleanup() if schema == 'ICN.fbs' and added != expected_boot_additions: unexplained += abs(added - expected_boot_additions) results[schema] = dict(before=before[schema], matched=matched, added=added, From e4d5e4bdd7dd67efca78319fb9436c4abdc7b258 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 13:03:49 -0500 Subject: [PATCH 19/31] Use the supported record scan page limit in rehearsal Request 50000 records per page while retaining complete snapshot and byte checks. Co-Authored-By: Codex --- deployment/scripts/roll-to-format5-rehearsal/README.md | 3 ++- deployment/scripts/roll-to-format5-rehearsal/records.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 4b5bfa24df..c5d2c56878 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -83,7 +83,8 @@ argument. The image and original copy are removed after the case. Record preservation uses the node's anonymous `/api/v1/data/scan` endpoint. The driver waits for startup connector persistence, then scans every standard -through the snapshot cursor. It checks the scan's advertised total against every +through the snapshot cursor, requesting the supported maximum of 50,000 records +per page. Page size does not change the completeness or byte checks. It checks the scan's advertised total against every record read. Stats counts sum source associations and can exceed unique record counts; both figures are retained in `scan-versus-stats.json`. Their equality is not a record-preservation condition. Before-state records stay in size-prefixed FlatBuffer streams in diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py index 4e0465618a..434242a08e 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/records.py +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -55,7 +55,7 @@ def request_frame(schema, cursor): struct.pack_into(' Date: Mon, 16 Nov 2026 13:12:46 -0500 Subject: [PATCH 20/31] Preserve repeated record occurrences in rehearsal comparisons Verify identical baseline bytes for repeated content addresses and require every occurrence after rollout. Extra or missing occurrences fail preservation. Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/README.md | 4 +-- .../roll-to-format5-rehearsal/records.py | 32 +++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index c5d2c56878..8b04e86ebe 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -92,8 +92,8 @@ is not a record-preservation condition. Before-state records stay in size-prefix stream offsets. Working indexes live in the container temporary filesystem; completed baseline indexes are copied into the evidence directory. Comparison uses a temporary copy so the baseline evidence stays unchanged. The after-state scan looks up every content address and -compares the complete bytes against that stream. Missing records, different -bytes, duplicate results and unexplained additions fail. Counts remain +compares the complete bytes against that stream. Missing occurrences, different +bytes, extra occurrences of an existing address and unexplained additions fail. Counts remain observations rather than the preservation criterion. The only allowed additions are startup ICN records: each must name a connector diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py index 434242a08e..6f51f5835d 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/records.py +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -138,15 +138,22 @@ def snapshot(directory, schemas): temporary = tempfile.TemporaryDirectory(prefix='record-snapshot-') index = Path(temporary.name) / 'index.sqlite' db = sqlite3.connect(index) - db.execute('CREATE TABLE records (cid BLOB PRIMARY KEY, offset INTEGER, size INTEGER, seen INTEGER DEFAULT 0) WITHOUT ROWID') + db.execute('CREATE TABLE records (cid BLOB PRIMARY KEY, offset INTEGER, size INTEGER, occurrences INTEGER DEFAULT 1, seen INTEGER DEFAULT 0) WITHOUT ROWID') db.execute('CREATE TABLE connectors (id BLOB PRIMARY KEY) WITHOUT ROWID') count = 0 - with (directory / (schema + '.sds')).open('wb') as stream: + with (directory / (schema + '.sds')).open('w+b') as stream: for data in scan(schema): cid = hashlib.sha256(data).digest() offset = stream.tell() - db.execute('INSERT INTO records(cid,offset,size) VALUES(?,?,?)', - (cid, offset, len(data))) + prior = db.execute('INSERT INTO records(cid,offset,size) VALUES(?,?,?) ' + 'ON CONFLICT(cid) DO UPDATE SET occurrences=occurrences+1 ' + 'RETURNING offset,size,occurrences', + (cid, offset, len(data))).fetchone() + if prior[2] > 1: + stream.seek(prior[0] + 4) + if prior[1] != len(data) or stream.read(prior[1]) != data: + raise RuntimeError('one content address has different baseline bytes') + stream.seek(offset) stream.write(struct.pack('= occurrences: + unexplained += 1 + db.execute('UPDATE records SET seen=seen+1 WHERE cid=?', (cid,)) matched += 1 else: added += 1 @@ -205,13 +214,16 @@ def compare(directory, schemas, boot_ms, expected_boot_additions): unexplained += 1 if (matched + added) % 20000 == 0: db.commit() - missing = db.execute('SELECT count(*) FROM records WHERE seen=0').fetchone()[0] + missing = db.execute('SELECT coalesce(sum(max(occurrences-seen,0)),0) FROM records').fetchone()[0] + distinct_before = db.execute('SELECT count(*) FROM records').fetchone()[0] db.commit() db.close() temporary.cleanup() if schema == 'ICN.fbs' and added != expected_boot_additions: unexplained += abs(added - expected_boot_additions) - results[schema] = dict(before=before[schema], matched=matched, added=added, + results[schema] = dict(before=before[schema], distinct_before=distinct_before, + baseline_duplicate_occurrences=before[schema]-distinct_before, + matched=matched, added=added, missing=missing, changed=changed, unexplained=unexplained) passed = all(not any(row.get(key, 0) for key in ('missing', 'changed', 'unexplained', 'unexpected_standard')) From 16e0567b1adfe33f91b155f45ba4d9127556dbed Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 13:49:14 -0500 Subject: [PATCH 21/31] Run the rehearsal Python driver on the build platform Use native Python with native systemd support while the release binaries retain their target architecture. Record assertions and scan limits are unchanged. Co-Authored-By: Codex --- deployment/scripts/roll-to-format5-rehearsal/Dockerfile | 3 ++- deployment/scripts/roll-to-format5-rehearsal/README.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/Dockerfile b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile index 5e4bbbf964..0bcfdb375d 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/Dockerfile +++ b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile @@ -1,6 +1,6 @@ FROM --platform=$BUILDPLATFORM ubuntu:24.04 AS init RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - systemd systemd-sysv dbus && rm -rf /var/lib/apt/lists/* + systemd systemd-sysv dbus python3 && rm -rf /var/lib/apt/lists/* FROM ubuntu:24.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -9,6 +9,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins # Native systemd launches emulated release binaries through the kernel handler. # Keep the target architecture's libraries alongside the native init libraries. COPY --from=init /usr/lib/ /usr/lib/ +COPY --from=init /usr/bin/python3.12 /usr/bin/python3.12 # Container init must not replace the host kernel's binary-format handlers. RUN ln -sf /dev/null /etc/systemd/system/systemd-binfmt.service && \ ln -sf /dev/null /etc/systemd/system/proc-sys-fs-binfmt_misc.mount diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 8b04e86ebe..e1ef5f874b 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -5,7 +5,7 @@ Docker with privileged containers, a cgroup mount usable by systemd, Python 3.12 or later on the driver, and enough disk for a case store and record evidence. Completed case stores and bundles are removed before the following case starts. The image uses Ubuntu 24.04 for the selected node architecture, amd64 by default. -Systemd and its support libraries run on the build platform's native architecture; +Systemd, the Python driver and their support libraries run on the build platform's native architecture; node binaries and target runtime libraries retain the selected architecture. This supports systemd service execution when the emulator rejects the amd64 systemd executor. Container binary-format management is masked so it cannot From 1e865afe5bd4a4f258ed7e641925b961ba5cec9f Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 13:51:22 -0500 Subject: [PATCH 22/31] Retain conversion observations before record comparison Save duration, sampled peak memory and switched size when the rollout exits so later read failures preserve the measurements. Co-Authored-By: Codex --- deployment/scripts/roll-to-format5-rehearsal/case.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 9f6cdbfc30..6918b53d2b 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -260,6 +260,10 @@ def run(case, minutes, seconds): time.sleep(0.25) code = proc.wait() (ROOT / 'roll.exit').write_text(str(code) + '\n') + (ROOT / 'conversion-observation.json').write_text(json.dumps({ + 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, + 'peak_conversion_rss_kib_sampled': peak_kib, + 'switched_fsql5_bytes': switched_bytes}) + '\n') ledger = (ROOT / 'ledger.log').read_text() wait_ready(seconds) if peer() != expected: From 63d7344ec9a83ab09c70508c3cb463929933fcb5 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 14:18:12 -0500 Subject: [PATCH 23/31] Retain original-binary selection proof before case cleanup Compare the selected binary with the baseline digest after recovery and precheck refusal, and preserve the result beside readiness and identity evidence. Co-Authored-By: Codex --- deployment/scripts/roll-to-format5-rehearsal/case.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 6918b53d2b..db50e6deb3 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -88,6 +88,8 @@ def tree_size(path): def run(case, minutes, seconds): if Path('/proc/1/comm').read_text().strip() != 'systemd': raise RuntimeError('PID 1 is not systemd') + with BIN.open('rb') as binary: + old_binary_sha256 = hashlib.file_digest(binary, 'sha256').hexdigest() if case == 'disk': # A real sparse loop filesystem bounds free space without filling a host disk. source = ROOT / 'store-source' @@ -268,6 +270,14 @@ def run(case, minutes, seconds): wait_ready(seconds) if peer() != expected: raise RuntimeError('identity changed') + if case != 'forward': + with BIN.open('rb') as binary: + selected_sha256 = hashlib.file_digest(binary, 'sha256').hexdigest() + (ROOT / 'bundle-selection.json').write_text(json.dumps({ + 'old_binary_selected': selected_sha256 == old_binary_sha256, + 'expected_sha256': old_binary_sha256, 'selected_sha256': selected_sha256}) + '\n') + if selected_sha256 != old_binary_sha256: + raise RuntimeError('the selected bundle is not the original binary') after_invocation, boot_additions = boot_observation() after = counts() (ROOT / 'counts-after.json').write_text(json.dumps(after, sort_keys=True) + '\n') From 70968d4d8c1915a69fabb8dc0c51c4f3c000b183 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 14:31:15 -0500 Subject: [PATCH 24/31] Document runtime observations and the exercised early retry Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/README.md | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index e1ef5f874b..c0f4125d9c 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -132,12 +132,20 @@ copies and creates format-5 marker bytes under `fsql4` with an empty store. The two installed old binaries, verified against their supplied digests, each reach readiness on their matching copied store in the amd64 rehearsal. -Both forward attempts fail before conversion because the verified v1.1.5 -linux/amd64 binary's runtime self-test reports -`stop-reaches-every-thread=false`. The command refuses with -`lacks the SDN runtime patches`. Other invocations report that flag true and -enter conversion, so the cause of the invocation-dependent refusal is not -established. The guard is not bypassed and node code is unchanged. +The preceding v1.1.5 forward attempts stopped at the runtime check. A subsequent +study of that same verified linux/amd64 archive ran 60 self-tests while idle and +60 during a real conversion. All idle invocations passed. Two concurrent +invocations reported `interruptible_aot=false` and `patched=false`, with no +reported errors; the other 58 passed. Every concurrent invocation was bracketed +by a running converter. The source checks whether cancellation stops every +spawned executor within a short observation window. These observations establish +intermittent failures in this environment, without isolating their cause. The +reported AOT speed ratio does not decide the check's result. + +The verified v1.1.6 archive also exercised the early conversion retry: its first +attempt refused the runtime patches after three seconds, and its second entered +conversion. The guard is not bypassed and node code is unchanged. Each retry +requires an empty format-5 destination and shares the original hard deadline. The first input's post-switch case verifies 45 partitions and reaches SWITCHED 1,414 seconds after CONVERTING. At that point the retained format-4 tree, From 5a58763c5c344b8266328fd63da96219d0a07ed2 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 15:26:13 -0500 Subject: [PATCH 25/31] Validate generated node profile refreshes and retain added frames Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/README.md | 11 ++++ .../scripts/roll-to-format5-rehearsal/case.py | 3 +- .../roll-to-format5-rehearsal/records.py | 66 ++++++++++++++++++- 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index c0f4125d9c..d2de3c50bd 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -106,6 +106,17 @@ provider/source pair from stored summaries and the source ledger and assigns a fresh `UPDATED_AT` timestamp. This accounts for startup records without allowing arbitrary changes to existing records. +The data API also merges the local node EPM into record reads. `epm.Service.Init` +rebuilds and persists that profile with a fresh signature timestamp. The +comparison recognizes only a Node EPM whose `/ipns/` address names the generated +test peer. After a service restart, one such profile may refresh its signature +and signature timestamp: every other byte, including all profile fields and +public keys, must match. The new timestamp must fall between rollout start and +the comparison. Other EPM records require exact address and byte preservation. +Before-state frames and every new or changed after-state frame remain in +size-prefixed streams under the private evidence directory. A profile refresh +has its own result counter and does not count as an exact record match. + Format-4 file hashes are taken at the script's explicit rehearsal barriers, after the original service stops and after the bundle is restored but before recovery starts that service. Only a newly added diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index db50e6deb3..4fd2320113 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -282,7 +282,8 @@ def run(case, minutes, seconds): after = counts() (ROOT / 'counts-after.json').write_text(json.dumps(after, sort_keys=True) + '\n') preservation = records.compare('/evidence/records-before', after, boot_ms, - boot_additions if after_invocation != before_invocation else 0) + boot_additions if after_invocation != before_invocation else 0, + expected, after_invocation != before_invocation) (ROOT / 'record-preservation.json').write_text(json.dumps(preservation, indent=2) + '\n') (ROOT / 'observations.json').write_text(json.dumps({ 'ready': True, 'running_peer_unchanged': True, 'counts_equal': before == after, diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py index 6f51f5835d..f3412433a2 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/records.py +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -10,6 +10,7 @@ import shutil import tempfile import struct +import time import urllib.request API = 'http://127.0.0.1:5001/api/v1/data/scan' @@ -44,6 +45,40 @@ def string(data, slot): return data[start + 4:start + 4 + size] +def local_profile(data, peer): + # EPM slots: MULTIFORMAT_ADDRESS=14, SIGNATURE=15, + # SIGNATURE_TIMESTAMP=16, ENTITY_TYPE=18 (Node=1). + prefix = b'' + if data[4:8] != b'$EPM' and data[8:12] == b'$EPM': + prefix, data = data[:4], data[4:] + if not peer or data[4:8] != b'$EPM' or number(data, 18, 'b') != 1: + return None + address = field(data, 14) + signature = field(data, 15) + timestamp = field(data, 16) + if not address or not signature or not timestamp: + return None + vector = address + scalar(data, address, 'I') + addresses = [] + for index in range(scalar(data, vector, 'I')): + offset = vector + 4 + index * 4 + start = offset + scalar(data, offset, 'I') + size = scalar(data, start, 'I') + addresses.append(data[start + 4:start + 4 + size]) + identity = peer.encode() + if b'/ipns/' + identity not in addresses: + return None + start = signature + scalar(data, signature, 'I') + size = scalar(data, start, 'I') + value = data[start + 4:start + 4 + size] + if not value or any(byte not in b'0123456789abcdefABCDEF' for byte in value): + return None + masked = bytearray(data) + masked[start + 4:start + 4 + size] = b'\0' * size + masked[timestamp:timestamp + 8] = b'\0' * 8 + return prefix + bytes(masked), scalar(data, timestamp, 'q') + + def request_frame(schema, cursor): # QRP slots: KIND=0 (Request, the default), SCHEMA_NAME=5, # LIMIT=20, CURSOR=21. A forward-built FlatBuffer needs no dependency. @@ -175,10 +210,13 @@ def snapshot(directory, schemas): return results -def compare(directory, schemas, boot_ms, expected_boot_additions): +def compare(directory, schemas, boot_ms, expected_boot_additions, + local_peer=None, allow_profile_refresh=False): directory = Path(directory) before = json.loads((directory / 'summary.json').read_text()) results = {} + additions = directory.parent / 'records-after-additions' + additions.mkdir(exist_ok=True) for schema in sorted(set(schemas) | set(before)): if schema not in before: results[schema] = {'unexpected_standard': True} @@ -188,8 +226,17 @@ def compare(directory, schemas, boot_ms, expected_boot_additions): shutil.copy2(directory / (schema + '.sqlite'), index) db = sqlite3.connect(index) added = changed = unexplained = matched = 0 + refreshed_profiles = 0 added_connectors = set() - with (directory / (schema + '.sds')).open('rb') as stream: + with (directory / (schema + '.sds')).open('rb') as stream, \ + (additions / (schema + '.sds')).open('wb') as after_stream: + profiles = [] + if schema == 'EPM.fbs' and allow_profile_refresh: + for old_cid, offset, size, occurrences in db.execute('SELECT cid,offset,size,occurrences FROM records'): + stream.seek(offset + 4) + profile = local_profile(stream.read(size), local_peer) + if profile and occurrences == 1: + profiles.append((old_cid, profile)) for data in scan(schema): cid = hashlib.sha256(data).digest() row = db.execute('SELECT offset,size,seen,occurrences FROM records WHERE cid=?', (cid,)).fetchone() @@ -198,18 +245,30 @@ def compare(directory, schemas, boot_ms, expected_boot_additions): stream.seek(offset + 4) if size != len(data) or stream.read(size) != data: changed += 1 + after_stream.write(struct.pack('= occurrences: unexplained += 1 db.execute('UPDATE records SET seen=seen+1 WHERE cid=?', (cid,)) matched += 1 else: added += 1 + after_stream.write(struct.pack(' Date: Mon, 16 Nov 2026 15:35:29 -0500 Subject: [PATCH 26/31] Check regenerated attestation signatures and timestamp fields Co-Authored-By: Codex --- .../roll-to-format5-rehearsal/README.md | 10 +++-- .../roll-to-format5-rehearsal/records.py | 43 ++++++++++++++++--- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index d2de3c50bd..b912d4aea3 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -109,10 +109,12 @@ allowing arbitrary changes to existing records. The data API also merges the local node EPM into record reads. `epm.Service.Init` rebuilds and persists that profile with a fresh signature timestamp. The comparison recognizes only a Node EPM whose `/ipns/` address names the generated -test peer. After a service restart, one such profile may refresh its signature -and signature timestamp: every other byte, including all profile fields and -public keys, must match. The new timestamp must fall between rollout start and -the comparison. Other EPM records require exact address and byte preservation. +test peer. After a service restart, one such profile may refresh its signature, +signature timestamp, and identity-attestation signatures and `issued_at` times. +`rebuildIdentityAttestationLocked` generates those attestations during startup. +Every other byte, including all profile fields, addresses, and public keys, must +match. Every new timestamp must fall between rollout start and the comparison. +Other EPM records require exact address and byte preservation. Before-state frames and every new or changed after-state frame remain in size-prefixed streams under the private evidence directory. A profile refresh has its own result counter and does not count as an exact record match. diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py index f3412433a2..97cd75e1f0 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/records.py +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -5,6 +5,7 @@ """ import hashlib import json +import re from pathlib import Path import sqlite3 import shutil @@ -21,8 +22,9 @@ def scalar(data, offset, fmt): return struct.unpack_from('<' + fmt, data, offset)[0] -def field(data, slot): - table = scalar(data, 0, 'I') +def field(data, slot, table=None): + if table is None: + table = scalar(data, 0, 'I') vtable = table - scalar(data, table, 'i') entry = 4 + slot * 2 if entry >= scalar(data, vtable, 'H'): @@ -76,7 +78,37 @@ def local_profile(data, peer): masked = bytearray(data) masked[start + 4:start + 4 + size] = b'\0' * size masked[timestamp:timestamp + 8] = b'\0' * 8 - return prefix + bytes(masked), scalar(data, timestamp, 'q') + issued_times = [] + proofs = field(data, 17) + if proofs: + vector = proofs + scalar(data, proofs, 'I') + for index in range(scalar(data, vector, 'I')): + entry = vector + 4 + index * 4 + table = entry + scalar(data, entry, 'I') + # ChainProof slots: SIGNATURE=4, SIGNED_PAYLOAD=5. + signature = field(data, 4, table) + payload = field(data, 5, table) + if not signature or not payload: + return None + start = signature + scalar(data, signature, 'I') + size = scalar(data, start, 'I') + if not size: + return None + masked[start + 4:start + 4 + size] = b'\0' * size + start = payload + scalar(data, payload, 'I') + size = scalar(data, start, 'I') + try: + decoded = bytes.fromhex(data[start + 4:start + 4 + size].decode('ascii')) + document = json.loads(decoded) + except (ValueError, UnicodeError): + return None + matches = list(re.finditer(rb'"issued_at":([0-9]+)', decoded)) + if len(matches) != 1 or document.get('issued_at') != int(matches[0][1]): + return None + issued_times.append(document['issued_at']) + first, last = matches[0].span(1) + masked[start + 4 + first * 2:start + 4 + last * 2] = b'0' * ((last - first) * 2) + return prefix + bytes(masked), scalar(data, timestamp, 'q'), tuple(issued_times) def request_frame(schema, cursor): @@ -264,7 +296,8 @@ def compare(directory, schemas, boot_ms, expected_boot_additions, old_cid, old_profile = profiles[0] unseen = db.execute('SELECT seen FROM records WHERE cid=?', (old_cid,)).fetchone()[0] == 0 if profile and unseen and profile[0] == old_profile[0] and \ - old_profile[1] <= boot_ms // 1000 <= profile[1] <= int(time.time()): + all(stamp <= boot_ms // 1000 for stamp in (old_profile[1], *old_profile[2])) and \ + all(boot_ms // 1000 <= stamp <= int(time.time()) for stamp in (profile[1], *profile[2])): db.execute('UPDATE records SET seen=seen+1 WHERE cid=?', (old_cid,)) refreshed_profiles += 1 else: @@ -289,4 +322,4 @@ def compare(directory, schemas, boot_ms, expected_boot_additions, ('missing', 'changed', 'unexplained', 'unexpected_standard')) for row in results.values()) return {'passed': passed, 'standards': results, - 'method': 'SHA-256 content address lookup and exact bytes; one generated local profile may refresh only its signature and signature timestamp'} + 'method': 'SHA-256 content address lookup and exact bytes; one generated local profile may refresh only signatures and their timestamps'} From ce33dd6981ee65b97a986d8cf95508b15b4969f8 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 17:29:36 -0500 Subject: [PATCH 27/31] Compare per-feed counts and clone case stores in the rehearsal The rehearsal reads every feed's count from the node's own stats before and after the roll and requires each to be unchanged, apart from the node's own startup connector records and a refreshed local profile. Case stores are copy-on-write clones of the source copy, the driver runs on Python 3.9, and each case container takes its CPU and memory limits from arguments. Co-Authored-By: Claude Opus 5.5 --- .../roll-to-format5-rehearsal/README.md | 16 +++- .../scripts/roll-to-format5-rehearsal/case.py | 76 ++++++++++++++++--- .../scripts/roll-to-format5-rehearsal/run.py | 46 +++++++++-- 3 files changed, 118 insertions(+), 20 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index b912d4aea3..7b4abe7c50 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -1,9 +1,11 @@ # Isolated systemd rehearsal This harness is an end-to-end test of `../roll-to-format5.sh`. It requires -Docker with privileged containers, a cgroup mount usable by systemd, Python 3.12 -or later on the driver, and enough disk for a case store and record evidence. Completed case stores -and bundles are removed before the following case starts. +Docker with privileged containers, a cgroup mount usable by systemd, Python 3.9 +or later on the driver, and enough disk for a case store and record evidence. Each case store is a +copy-on-write clone of the source copy where the filesystem supports it (`cp -c` on APFS, +`--reflink=auto` elsewhere). Completed case stores and bundles are removed before the following +case starts. `--cpus` and `--memory` set each case container's limits (2 CPUs and 6 GB by default). The image uses Ubuntu 24.04 for the selected node architecture, amd64 by default. Systemd, the Python driver and their support libraries run on the build platform's native architecture; node binaries and target runtime libraries retain the selected architecture. @@ -96,6 +98,14 @@ compares the complete bytes against that stream. Missing occurrences, different bytes, extra occurrences of an existing address and unexplained additions fail. Counts remain observations rather than the preservation criterion. +Per-feed counts come from the node's own `/api/v1/stats` (`sources[]`, one row +per standard, provider, source and batch), read before and after from a stats +build that is not stale, was made after the node's startup writes and gives the +same counts twice. `feeds-before.json`, `feeds-after.json` and +`feed-counts-comparison.json` hold them. Every feed must keep its count; the only +differences allowed are `$ICN` feeds whose total rise equals the startup +persistence count, and an unchanged `$EPM` total when the local profile refreshed. + The only allowed additions are startup ICN records: each must name a connector present before, have an update timestamp after the rollout began, and occur once per connector. Their number must equal the current invocation's logged diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 4fd2320113..8a451f2aa4 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Run inside an isolated container whose PID 1 is systemd.""" +import calendar import hashlib import json import os @@ -48,14 +49,64 @@ def hashes(): return result -def counts(): +def stats_read(): with urllib.request.urlopen('http://127.0.0.1:5001/api/v1/stats', timeout=60) as r: - data = json.load(r) + return json.load(r) + + +def feeds(data): + # One row per feed: (standard, provider, source, batch) with its count. + return {'|'.join((f['schema'], f.get('provider_id', ''), f.get('source_name', ''), + f.get('batch_id', ''))): f['count'] for f in data.get('sources', [])} + + +def counts(name, after_unix): + # The stats lane rebuilds every few seconds; take two equal readings of a + # build that is not stale and was made after the node's startup writes. + end = time.monotonic() + 600 + previous = None + while time.monotonic() < end: + data = stats_read() + as_of = data.get('as_of') + fresh = not data.get('stale') and (as_of is None or + calendar.timegm(time.strptime(as_of, '%Y-%m-%dT%H:%M:%SZ')) > after_unix) + reading = ({s['schema']: s['count'] for s in data['schemas']}, feeds(data)) + if fresh and reading == previous: + break + previous = reading if fresh else None + time.sleep(6) + else: + raise RuntimeError('stats did not settle on a fresh build') + result, per_feed = reading + if not result or not sum(result.values()) or not per_feed: + raise RuntimeError('public stats has no populated per-standard or per-feed counts') # Persist only aggregates, never source rows or records. - result = {s['schema']: s['count'] for s in data['schemas']} - if not result or not sum(result.values()): - raise RuntimeError('public stats has no populated per-standard counts') - return result + (ROOT / ('feeds-' + name + '.json')).write_text(json.dumps(per_feed, indent=1, sort_keys=True) + '\n') + return result, per_feed + + +def compare_feeds(before, after, boot_additions, profile_refreshes): + # Every feed keeps its count. The node's own startup writes are the only + # differences allowed: connector records ($ICN) equal to the persisted + # count and a refreshed local profile ($EPM), which replaces itself. + rows = {} + icn_delta = 0 + for key in sorted(set(before) | set(after)): + b, a = before.get(key), after.get(key) + if a == b: + continue + rows[key] = {'before': b, 'after': a} + if key.startswith('ICN.fbs|'): + icn_delta += (a or 0) - (b or 0) + unexplained = [k for k in rows if not k.startswith('ICN.fbs|')] + epm = [k for k in unexplained if k.startswith('EPM.fbs|')] + if profile_refreshes and epm and sum((rows[k]['after'] or 0) - (rows[k]['before'] or 0) for k in epm) == 0: + unexplained = [k for k in unexplained if k not in epm] + passed = not unexplained and icn_delta == boot_additions + return {'passed': passed, 'feeds_before': len(before), 'feeds_after': len(after), + 'feeds_equal': len(set(before) & set(after)) - len(rows), 'differences': rows, + 'icn_delta': icn_delta, 'startup_icn_persisted': boot_additions, + 'unexplained': unexplained} def peer(): @@ -167,7 +218,7 @@ def run(case, minutes, seconds): (ROOT / 'baseline.json').write_text(json.dumps({'ready': True, 'http_status': 200}) + '\n') expected = peer() before_invocation, _ = boot_observation() - before = counts() + before, feeds_before = counts('before', time.time()) (ROOT / 'counts-before.json').write_text(json.dumps(before, sort_keys=True) + '\n') records.snapshot('/evidence/records-before', before) before_hashes = None @@ -279,18 +330,23 @@ def run(case, minutes, seconds): if selected_sha256 != old_binary_sha256: raise RuntimeError('the selected bundle is not the original binary') after_invocation, boot_additions = boot_observation() - after = counts() + after, feeds_after = counts('after', time.time()) (ROOT / 'counts-after.json').write_text(json.dumps(after, sort_keys=True) + '\n') preservation = records.compare('/evidence/records-before', after, boot_ms, boot_additions if after_invocation != before_invocation else 0, expected, after_invocation != before_invocation) (ROOT / 'record-preservation.json').write_text(json.dumps(preservation, indent=2) + '\n') + refreshes = sum(row.get('refreshed_local_profiles', 0) for row in preservation['standards'].values()) + feed_result = compare_feeds(feeds_before, feeds_after, + boot_additions if after_invocation != before_invocation else 0, refreshes) + (ROOT / 'feed-counts-comparison.json').write_text(json.dumps(feed_result, indent=2) + '\n') (ROOT / 'observations.json').write_text(json.dumps({ 'ready': True, 'running_peer_unchanged': True, 'counts_equal': before == after, 'roll_exit': code, 'injected': injected, 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, 'peak_conversion_rss_kib_sampled': peak_kib, 'switched_fsql5_bytes': switched_bytes, - 'stopped_format4_hashes': stopped_hash_result, 'records_preserved': preservation['passed']}) + '\n') + 'stopped_format4_hashes': stopped_hash_result, 'records_preserved': preservation['passed'], + 'feed_counts_preserved': feed_result['passed']}) + '\n') if any(' ' + event + ' ' in ledger for event in ('SUCCESS', 'ROLLED_BACK_BEFORE_SWITCH', 'ROLLED_BACK_AFTER_SWITCH')): replay_pid = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) @@ -304,6 +360,8 @@ def run(case, minutes, seconds): raise RuntimeError('second run was not refused without restarting the node') if not preservation['passed']: raise RuntimeError('record preservation failed') + if not feed_result['passed']: + raise RuntimeError('per-feed counts changed') if case == 'forward': if code != 0 or ' SUCCESS ' not in ledger: raise RuntimeError('forward conversion failed') diff --git a/deployment/scripts/roll-to-format5-rehearsal/run.py b/deployment/scripts/roll-to-format5-rehearsal/run.py index 03df9d82b3..006e623ae1 100755 --- a/deployment/scripts/roll-to-format5-rehearsal/run.py +++ b/deployment/scripts/roll-to-format5-rehearsal/run.py @@ -14,8 +14,16 @@ 'disk', 'script-term', 'script-hup') +def sha256(path): + digest = hashlib.sha256() + with open(path, 'rb') as stream: + for block in iter(lambda: stream.read(1 << 20), b''): + digest.update(block) + return digest.hexdigest() + + def verify(archive, checksums): - digest = hashlib.file_digest(open(archive, 'rb'), 'sha256').hexdigest() + digest = sha256(archive) expected = [line.split()[0] for line in checksums.read_text().splitlines() if len(line.split()) == 2 and line.split()[1].lstrip('*') == archive.name] if expected != [digest]: @@ -30,7 +38,13 @@ def unpack(archive, destination): tops = {m.name.split('/')[0] for m in members if m.name} if len(tops) != 1: raise ValueError('archive needs one top-level bundle directory') - bundle.extractall(destination.parent / (destination.name + '.unpack'), filter='data') + for member in members: + if member.name.startswith('/') or '..' in member.name.split('/') or \ + not (member.isfile() or member.isdir() or member.issym()): + raise ValueError('archive member outside the bundle: ' + member.name) + unpacked = destination.parent / (destination.name + '.unpack') + unpacked.mkdir() + subprocess.run(['tar', '-xzf', str(archive), '--no-same-owner', '-C', str(unpacked)], check=True) source = destination.parent / (destination.name + '.unpack') / tops.pop() for child in source.iterdir(): shutil.move(str(child), destination / child.name) @@ -40,8 +54,23 @@ def unpack(archive, destination): raise ValueError('archive has no bundle launcher') -def ignore_private(directory, names): - return [n for n in names if n in ('keys', 'secrets', 'auth.db') or 'udl' in n.lower()] +def private(name): + return name in ('keys', 'secrets', 'auth.db') or 'udl' in name.lower() + + +def copy_store(source, destination): + # A copy-on-write clone where the filesystem supports one, then remove + # private material, which never takes part in a rehearsal. + clone = ['cp', '-c', '-p', '-R', str(source), str(destination)] if sys.platform == 'darwin' \ + else ['cp', '-a', '--reflink=auto', str(source), str(destination)] + subprocess.run(clone, check=True) + for directory, names, files in os.walk(destination, topdown=True): + for name in [n for n in names if private(n)]: + shutil.rmtree(os.path.join(directory, name)) + names.remove(name) + for name in files: + if private(name): + os.unlink(os.path.join(directory, name)) def main(): @@ -57,6 +86,8 @@ def main(): ap.add_argument('--case', choices=CASES, action='append') ap.add_argument('--conversion-minutes', type=int, default=90) ap.add_argument('--ready-seconds', type=int, default=300) + ap.add_argument('--cpus', default='2') + ap.add_argument('--memory', default='6g') args = ap.parse_args() if len(args.old_binary) != len(args.store) or len(args.old_sha256) != len(args.store): ap.error('each store requires its matching old binary and SHA-256') @@ -71,8 +102,7 @@ def main(): try: verified = [verify(args.new_archive, args.new_checksums)] for binary, expected in zip(args.old_binary, args.old_sha256): - with binary.open('rb') as stream: - digest = hashlib.file_digest(stream, 'sha256').hexdigest() + digest = sha256(binary) if digest != expected: raise ValueError('old binary checksum mismatch') verified.append({'old_binary': binary.name, 'sha256': digest}) @@ -87,7 +117,7 @@ def main(): for case in args.case or CASES: directory = work / ('store-' + str(number)) / case directory.mkdir(parents=True) - shutil.copytree(source, directory / 'store', ignore=ignore_private) + copy_store(source, directory / 'store') old_bundle = directory / 'base/sdn-bundle' (old_bundle / 'bin').mkdir(parents=True) old_binary = args.old_binary[number - 1] @@ -103,7 +133,7 @@ def main(): with open(directory / 'container.log', 'w') as log: subprocess.run(['docker', 'run', '-d', '-t', '--name', active, '--hostname=sdn-rehearsal', '--privileged', '--cgroupns=host', '--network=none', '--platform=linux/' + args.arch, - '--cpus=2', '--memory=6g', '--tmpfs=/run', '--tmpfs=/run/lock', + '--cpus=' + args.cpus, '--memory=' + args.memory, '--tmpfs=/run', '--tmpfs=/run/lock', '--tmpfs=/tmp', '-v', '/sys/fs/cgroup:/sys/fs/cgroup:rw', '-v', str(cache) + ':/engine-cache', '-v', str(evidence) + ':/evidence', From 6901ece2cef7ba79613f73754ccbd26a2c8041c7 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 20:07:55 -0500 Subject: [PATCH 28/31] Check the converted store over a clean stop and start in the rehearsal After a forward roll, the rehearsal stops and starts the new node once, fails on any quarantined or P4_E_CORRUPT journal line from the stop on, and repeats the record-by-record and per-feed comparisons against the format-4 baseline. Each node start since the roll may add one connector record per connector. Co-Authored-By: Claude Opus 5.5 --- .../roll-to-format5-rehearsal/README.md | 2 +- .../scripts/roll-to-format5-rehearsal/case.py | 44 +++++++++++++++++++ .../roll-to-format5-rehearsal/records.py | 12 ++--- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 7b4abe7c50..8d1c08de40 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -69,7 +69,7 @@ silently create an empty store. | Case | Required evidence | | --- | --- | -| forward | Zero exit, SUCCESS, record-by-record preservation, ready and unchanged identity. A second invocation is refused. | +| forward | Zero exit, SUCCESS, record-by-record preservation, ready and unchanged identity. A second invocation is refused. Then one clean stop and start of the new node: no `quarantined` or `P4_E_CORRUPT` line in its journal from the stop on, and record-by-record and per-feed preservation again against the format-4 baseline (`restart-check.json`, `record-preservation-after-restart.json`, `feed-counts-comparison-after-restart.json`). Each start since the roll may add one `$ICN` record per connector. | | interrupted | SIGTERM to the recorded migration PID after the pending directory appears, automatic recovery, ready, unchanged identity and identical fsql4 file hashes. | | budget | One-minute conversion deadline, automatic recovery and identical fsql4 file hashes. A fast conversion that completes inside the budget fails this injection case. | | after-switch | A unit pre-start condition rejects the new fsql5 selection. The original bundle restarts after recovery removes that selection. | diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 8a451f2aa4..4a61dec765 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -132,6 +132,44 @@ def boot_observation(): raise RuntimeError('connector startup persistence did not finish') +def restart_check(seconds, expected, feeds_before, boot_ms, before_invocation, + first_invocation, first_additions): + # One clean stop and start of the new node on the converted store, then + # the same record and feed comparisons against the format-4 baseline. + stop_unix = time.time() + command('systemctl', 'stop', UNIT) + if command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) != '0': + raise RuntimeError('the new node did not stop') + command('systemctl', 'start', '--no-block', UNIT) + wait_ready(seconds) + if peer() != expected: + raise RuntimeError('identity changed over the restart') + invocation, additions = boot_observation() + if invocation == first_invocation: + raise RuntimeError('the node did not restart') + since = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(stop_unix - 1)) + journal = subprocess.run(['journalctl', '--no-pager', '-o', 'cat', '--utc', '--since', since, + '-u', UNIT], capture_output=True, text=True, check=True).stdout + corrupt = [line[:300] for line in journal.splitlines() + if 'quarantined' in line or 'P4_E_CORRUPT' in line] + (ROOT / 'restart-journal-corrupt-lines.json').write_text(json.dumps(corrupt, indent=1) + '\n') + after, feeds_after = counts('after-restart', time.time()) + (ROOT / 'counts-after-restart.json').write_text(json.dumps(after, sort_keys=True) + '\n') + preservation = records.compare('/evidence/records-before', after, boot_ms, + first_additions + additions, expected, True, starts=2, + additions_name='records-after-restart-additions') + (ROOT / 'record-preservation-after-restart.json').write_text(json.dumps(preservation, indent=2) + '\n') + refreshes = sum(row.get('refreshed_local_profiles', 0) for row in preservation['standards'].values()) + feed_result = compare_feeds(feeds_before, feeds_after, first_additions + additions, refreshes) + (ROOT / 'feed-counts-comparison-after-restart.json').write_text(json.dumps(feed_result, indent=2) + '\n') + result = {'passed': not corrupt and preservation['passed'] and feed_result['passed'], + 'corrupt_journal_lines': len(corrupt), 'records_preserved': preservation['passed'], + 'feed_counts_preserved': feed_result['passed'], + 'startup_icn_persisted': [first_additions, additions]} + (ROOT / 'restart-check.json').write_text(json.dumps(result, indent=2) + '\n') + return result + + def tree_size(path): return sum(p.stat().st_size for p in path.rglob('*') if p.is_file()) if path.exists() else 0 @@ -358,10 +396,16 @@ def run(case, minutes, seconds): 'refused': second != 0, 'daemon_pid_unchanged': unchanged}) + '\n') if second == 0 or not unchanged: raise RuntimeError('second run was not refused without restarting the node') + restart_result = None + if case == 'forward' and code == 0 and ' SUCCESS ' in ledger: + restart_result = restart_check(seconds, expected, feeds_before, boot_ms, + before_invocation, after_invocation, boot_additions) if not preservation['passed']: raise RuntimeError('record preservation failed') if not feed_result['passed']: raise RuntimeError('per-feed counts changed') + if restart_result is not None and not restart_result['passed']: + raise RuntimeError('the converted store changed over a clean stop and start') if case == 'forward': if code != 0 or ' SUCCESS ' not in ledger: raise RuntimeError('forward conversion failed') diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py index 97cd75e1f0..62f9ade74d 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/records.py +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -243,11 +243,12 @@ def snapshot(directory, schemas): def compare(directory, schemas, boot_ms, expected_boot_additions, - local_peer=None, allow_profile_refresh=False): + local_peer=None, allow_profile_refresh=False, starts=1, + additions_name='records-after-additions'): directory = Path(directory) before = json.loads((directory / 'summary.json').read_text()) results = {} - additions = directory.parent / 'records-after-additions' + additions = directory.parent / additions_name additions.mkdir(exist_ok=True) for schema in sorted(set(schemas) | set(before)): if schema not in before: @@ -259,7 +260,7 @@ def compare(directory, schemas, boot_ms, expected_boot_additions, db = sqlite3.connect(index) added = changed = unexplained = matched = 0 refreshed_profiles = 0 - added_connectors = set() + added_connectors = {} with (directory / (schema + '.sds')).open('rb') as stream, \ (additions / (schema + '.sds')).open('wb') as after_stream: profiles = [] @@ -288,9 +289,10 @@ def compare(directory, schemas, boot_ms, expected_boot_additions, if schema == 'ICN.fbs': connector, updated = icn_fields(data) known = db.execute('SELECT 1 FROM connectors WHERE id=?', (connector,)).fetchone() - if not known or updated < boot_ms or connector in added_connectors: + # Each node start since the roll writes one record per connector. + if not known or updated < boot_ms or added_connectors.get(connector, 0) >= starts: unexplained += 1 - added_connectors.add(connector) + added_connectors[connector] = added_connectors.get(connector, 0) + 1 elif schema == 'EPM.fbs' and len(profiles) == 1 and not refreshed_profiles: profile = local_profile(data, local_peer) old_cid, old_profile = profiles[0] From 8ff09256873581b5434c433257fa915d0db18a98 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 22:07:58 -0500 Subject: [PATCH 29/31] Bound per-feed connector rises and check per-standard counts Startup connector records raise the per-standard $ICN count by exactly the persisted count while the per-feed rows can stay unchanged, because they can land in the local lane. Per-feed $ICN rows may rise by at most that count; per-standard counts must be equal apart from that $ICN rise. Co-Authored-By: Claude Opus 5.5 --- .../roll-to-format5-rehearsal/README.md | 7 +++-- .../scripts/roll-to-format5-rehearsal/case.py | 27 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 8d1c08de40..35c836682b 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -103,8 +103,11 @@ per standard, provider, source and batch), read before and after from a stats build that is not stale, was made after the node's startup writes and gives the same counts twice. `feeds-before.json`, `feeds-after.json` and `feed-counts-comparison.json` hold them. Every feed must keep its count; the only -differences allowed are `$ICN` feeds whose total rise equals the startup -persistence count, and an unchanged `$EPM` total when the local profile refreshed. +differences allowed are `$ICN` feeds, whose total rise is at most the startup +persistence count (startup connector records can land in the local lane, which +stats does not list per feed), and an unchanged `$EPM` total when the local +profile refreshed. The per-standard counts must be equal apart from `$ICN`, which +must rise by exactly the startup persistence count. The only allowed additions are startup ICN records: each must name a connector present before, have an update timestamp after the rollout began, and occur diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py index 4a61dec765..6ad7cb34c5 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/case.py +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -87,8 +87,11 @@ def counts(name, after_unix): def compare_feeds(before, after, boot_additions, profile_refreshes): # Every feed keeps its count. The node's own startup writes are the only - # differences allowed: connector records ($ICN) equal to the persisted - # count and a refreshed local profile ($EPM), which replaces itself. + # differences allowed: connector records ($ICN), whose feed rows rise by at + # most the persisted count (startup records can land in the local lane, + # which stats does not list per feed), and a refreshed local profile ($EPM), + # which replaces itself. The per-standard $ICN count must rise by exactly + # the persisted count; compare_standards checks that. rows = {} icn_delta = 0 for key in sorted(set(before) | set(after)): @@ -102,7 +105,7 @@ def compare_feeds(before, after, boot_additions, profile_refreshes): epm = [k for k in unexplained if k.startswith('EPM.fbs|')] if profile_refreshes and epm and sum((rows[k]['after'] or 0) - (rows[k]['before'] or 0) for k in epm) == 0: unexplained = [k for k in unexplained if k not in epm] - passed = not unexplained and icn_delta == boot_additions + passed = not unexplained and 0 <= icn_delta <= boot_additions return {'passed': passed, 'feeds_before': len(before), 'feeds_after': len(after), 'feeds_equal': len(set(before) & set(after)) - len(rows), 'differences': rows, 'icn_delta': icn_delta, 'startup_icn_persisted': boot_additions, @@ -132,6 +135,16 @@ def boot_observation(): raise RuntimeError('connector startup persistence did not finish') +def compare_standards(before, after, boot_additions, profile_refreshes): + rows = {k: {'before': before.get(k), 'after': after.get(k)} + for k in sorted(set(before) | set(after)) if before.get(k) != after.get(k)} + icn = (after.get('ICN.fbs') or 0) - (before.get('ICN.fbs') or 0) + unexplained = [k for k in rows if k != 'ICN.fbs' and not (k == 'EPM.fbs' and profile_refreshes + and before.get(k) == after.get(k))] + return {'passed': not unexplained and icn == boot_additions, 'differences': rows, + 'icn_delta': icn, 'startup_icn_persisted': boot_additions, 'unexplained': unexplained} + + def restart_check(seconds, expected, feeds_before, boot_ms, before_invocation, first_invocation, first_additions): # One clean stop and start of the new node on the converted store, then @@ -161,6 +174,10 @@ def restart_check(seconds, expected, feeds_before, boot_ms, before_invocation, (ROOT / 'record-preservation-after-restart.json').write_text(json.dumps(preservation, indent=2) + '\n') refreshes = sum(row.get('refreshed_local_profiles', 0) for row in preservation['standards'].values()) feed_result = compare_feeds(feeds_before, feeds_after, first_additions + additions, refreshes) + standard_result = compare_standards(json.loads((ROOT / 'counts-before.json').read_text()), after, + first_additions + additions, refreshes) + feed_result['standards'] = standard_result + feed_result['passed'] = feed_result['passed'] and standard_result['passed'] (ROOT / 'feed-counts-comparison-after-restart.json').write_text(json.dumps(feed_result, indent=2) + '\n') result = {'passed': not corrupt and preservation['passed'] and feed_result['passed'], 'corrupt_journal_lines': len(corrupt), 'records_preserved': preservation['passed'], @@ -377,6 +394,10 @@ def run(case, minutes, seconds): refreshes = sum(row.get('refreshed_local_profiles', 0) for row in preservation['standards'].values()) feed_result = compare_feeds(feeds_before, feeds_after, boot_additions if after_invocation != before_invocation else 0, refreshes) + standard_result = compare_standards(before, after, + boot_additions if after_invocation != before_invocation else 0, refreshes) + feed_result['standards'] = standard_result + feed_result['passed'] = feed_result['passed'] and standard_result['passed'] (ROOT / 'feed-counts-comparison.json').write_text(json.dumps(feed_result, indent=2) + '\n') (ROOT / 'observations.json').write_text(json.dumps({ 'ready': True, 'running_peer_unchanged': True, 'counts_equal': before == after, From 9a494869d09472d28f2720f996b251b4fb7c1917 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 22:27:04 -0500 Subject: [PATCH 30/31] Give each rehearsal record scan page fifteen minutes A 50,000-record scan page of an emulated node under load took longer than two minutes. No comparison changes. Co-Authored-By: Claude Opus 5.5 --- deployment/scripts/roll-to-format5-rehearsal/records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py index 62f9ade74d..37a59c0623 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/records.py +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -152,7 +152,7 @@ def scan(schema): request = urllib.request.Request(API, request_frame(schema, cursor), {'Content-Type': MEDIA}) next_cursor = None - with urllib.request.urlopen(request, timeout=120) as response: + with urllib.request.urlopen(request, timeout=900) as response: while True: header = read_frame(response) if header is None: From 5fb3b5cb990b745b4afce684ab7e1786119ea6c2 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Tue, 17 Nov 2026 01:20:31 -0500 Subject: [PATCH 31/31] Record the v1.1.6 rehearsal results on both store copies Co-Authored-By: Claude Opus 5.5 --- .../scripts/roll-to-format5-rehearsal/README.md | 13 +++++++++---- deployment/scripts/roll-to-format5.md | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md index 35c836682b..8d6a9d828f 100644 --- a/deployment/scripts/roll-to-format5-rehearsal/README.md +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -190,7 +190,12 @@ those measurements; they require a completed rehearsal before they establish preservation. The earlier failing results remain evidence of those earlier measurements. -The script is not cleared for deployment. Recovery ledger events alone do not -establish the required preservation proof or a successful new-node forward run. -Each retained case result and exit file determines whether the complete case -passed; an intended fault that was never injected is not a tested recovery leg. +The v1.1.6 rehearsal on both 2026-11-15 production store copies passed the +forward case with the restart check (host-01 copy with the linux/amd64 release: +4,419,593 records and 8,959 feeds preserved over the roll and over a clean stop +and start, no quarantine line; host-02 copy with the linux/arm64 build of the +same release as the new node, because its emulated amd64 conversion exceeded +180 minutes: every record of 13 standards and 670 feeds preserved, no quarantine +line) and the failed-conversion case (format-4 file hashes unchanged, every +record present after the return) on both copies. Earlier failing results +remain evidence of the measurements they made. diff --git a/deployment/scripts/roll-to-format5.md b/deployment/scripts/roll-to-format5.md index f25f03784d..711a31422d 100644 --- a/deployment/scripts/roll-to-format5.md +++ b/deployment/scripts/roll-to-format5.md @@ -6,7 +6,7 @@ recovery. The script requires Linux, Bash, GNU coreutils, systemd and curl, and runs as root. The service also runs as root. No schema or record data is printed by the script. Conversion output comes from the node binary. -The release-source recovery contract is verified against v1.1.5. End-to-end +The release-source recovery contract is verified against v1.1.5 and v1.1.6. End-to-end validation on production-store copies is required before deployment. The included rehearsal installs systemd from official package archives; syntax checks do not establish recovery safety.