diff --git a/deployment/scripts/roll-to-format5-rehearsal/Dockerfile b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile new file mode 100644 index 0000000000..0bcfdb375d --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/Dockerfile @@ -0,0 +1,18 @@ +FROM --platform=$BUILDPLATFORM ubuntu:24.04 AS init +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + systemd systemd-sysv dbus python3 && rm -rf /var/lib/apt/lists/* + +FROM ubuntu:24.04 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + systemd systemd-sysv dbus curl ca-certificates procps python3 && \ + rm -rf /var/lib/apt/lists/* +# Native systemd launches emulated release binaries through the kernel handler. +# Keep the target architecture's libraries alongside the native init libraries. +COPY --from=init /usr/lib/ /usr/lib/ +COPY --from=init /usr/bin/python3.12 /usr/bin/python3.12 +# Container init must not replace the host kernel's binary-format handlers. +RUN ln -sf /dev/null /etc/systemd/system/systemd-binfmt.service && \ + ln -sf /dev/null /etc/systemd/system/proc-sys-fs-binfmt_misc.mount +ENV container=docker +STOPSIGNAL SIGRTMIN+3 +CMD ["/lib/systemd/systemd", "--system", "--unit=multi-user.target"] diff --git a/deployment/scripts/roll-to-format5-rehearsal/README.md b/deployment/scripts/roll-to-format5-rehearsal/README.md new file mode 100644 index 0000000000..8d6a9d828f --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/README.md @@ -0,0 +1,201 @@ +# Isolated systemd rehearsal + +This harness is an end-to-end test of `../roll-to-format5.sh`. It requires +Docker with privileged containers, a cgroup mount usable by systemd, Python 3.9 +or later on the driver, and enough disk for a case store and record evidence. Each case store is a +copy-on-write clone of the source copy where the filesystem supports it (`cp -c` on APFS, +`--reflink=auto` elsewhere). Completed case stores and bundles are removed before the following +case starts. `--cpus` and `--memory` set each case container's limits (2 CPUs and 6 GB by default). +The image uses Ubuntu 24.04 for the selected node architecture, amd64 by default. +Systemd, the Python driver and their support libraries run on the build platform's native architecture; +node binaries and target runtime libraries retain the selected architecture. +This supports systemd service execution when the emulator rejects the amd64 +systemd executor. Container binary-format management is masked so it cannot +replace the Docker kernel's emulator registrations. + +The Dockerfile installs systemd, D-Bus, curl, Python and supporting utilities +from Ubuntu's official package archives. Package downloads occur only during +the image build. Image-build failure is a blocked rehearsal. Each source store's old binary and SHA-256, plus the new release archive and +its release checksum file, are explicit inputs; checksum mismatch aborts +before store copies or containers are made. + +```bash +python3 run.py \ + --old-binary "$FIRST_OLD_BINARY" --old-sha256 "$FIRST_OLD_SHA256" \ + --old-binary "$SECOND_OLD_BINARY" --old-sha256 "$SECOND_OLD_SHA256" \ + --new-archive "$NEW_ARCHIVE" --new-checksums "$NEW_CHECKSUMS" \ + --arch amd64 \ + --store "$FIRST_STORE_COPY" --store "$SECOND_STORE_COPY" \ + --work-dir "$NEW_REHEARSAL_DIRECTORY" \ + --record-evidence-dir "$PRIVATE_RECORD_EVIDENCE_DIRECTORY" \ + --conversion-minutes 90 --ready-seconds 300 +``` + +Use the binary matching each store's installed build as OLD and the published +selected release archive as NEW. Each old binary has its bundle's `manifest.json` beside it. +The old bundle contains `bin/spacedatanetwork` and that manifest; the new bundle +retains the release archive's launcher and runtime layout. Old binary arguments +and their digests follow the order of the store arguments. +Changing `--new-archive` selects another release, with its corresponding checksum +file supplied through `--new-checksums`. The output directory must not exist. +A selective repeat uses `--case` with one or more case names. +Every repeat uses a fresh output directory, evidence directory and store copy. +Runtime prechecks and early migration runtime refusals use the script's bounded +retry policy. Exhausted retries remain failures; a repeat does not alter results. +Start Docker before +running and quit it after the run. The harness stops and removes only containers +it creates. It retains case logs, hashes, ledgers and results. Completed case +stores, backup directories, bundles and disk images are removed; record evidence +stays in the explicitly supplied private evidence directory. + +The source stores are read only. Copies omit `keys`, `secrets` and `auth.db`. +Every case starts its old node from a fresh copy, with a generated test identity, +a generated test password, an engine cache separate from identity material, a stable container hostname, +`Restart=always` with `RestartSec=15`, loopback listeners, disabled DHT and mDNS, no trusted peers, no host ports and +`--network=none`. The stable hostname allows the release to create a durable +test identity without Docker's ephemeral-hostname refusal. These releases +replace an empty bootstrap list with public defaults. A synthetic peer identity +at unused loopback port 9 suppresses that fallback without naming a real peer. +Docker's network namespace has no external interface. The loopback admin +listener has `require_auth: false` to expose aggregate stats without credentials. +Provider flow services are explicitly empty, so prewarm checks only the +engine artifacts required by the isolated node. The running peer identity comes from the current systemd invocation journal. +`--cache-dir` reuses compiled engine artifacts across cases; identity material +is never placed in that directory. The new release's engine +is prewarmed before stopping the old node. No runtime permission patch or fake +systemctl is used. A failure to start the old release is recorded as a failed +case and requires inspection; the harness does not replace the release or +silently create an empty store. + +| Case | Required evidence | +| --- | --- | +| forward | Zero exit, SUCCESS, record-by-record preservation, ready and unchanged identity. A second invocation is refused. Then one clean stop and start of the new node: no `quarantined` or `P4_E_CORRUPT` line in its journal from the stop on, and record-by-record and per-feed preservation again against the format-4 baseline (`restart-check.json`, `record-preservation-after-restart.json`, `feed-counts-comparison-after-restart.json`). Each start since the roll may add one `$ICN` record per connector. | +| interrupted | SIGTERM to the recorded migration PID after the pending directory appears, automatic recovery, ready, unchanged identity and identical fsql4 file hashes. | +| budget | One-minute conversion deadline, automatic recovery and identical fsql4 file hashes. A fast conversion that completes inside the budget fails this injection case. | +| after-switch | A unit pre-start condition rejects the new fsql5 selection. The original bundle restarts after recovery removes that selection. | +| wrong-peer | Wrong expected identity is refused before stopping, with unchanged PID, preserved records and bundle/store selection. | +| disk | Insufficient free bytes on the supplied filesystem are refused before stopping. A bounded ext4 loop filesystem supplies the space constraint. | +| script-term / script-hup | Signal to the exact roll-script MainPID during conversion invokes automatic recovery. | + +The disk case creates a sparse ext4 image with capacity equal to the copied +store size plus 2 GiB, mounts it with a loop device, and copies the store into +that filesystem before startup. It checks the real free-space result against +the production formula. It does not fill a host disk or add a test-only script +argument. The image and original copy are removed after the case. + +Record preservation uses the node's anonymous `/api/v1/data/scan` endpoint. +The driver waits for startup connector persistence, then scans every standard +through the snapshot cursor, requesting the supported maximum of 50,000 records +per page. Page size does not change the completeness or byte checks. It checks the scan's advertised total against every +record read. Stats counts sum source associations and can exceed unique record +counts; both figures are retained in `scan-versus-stats.json`. Their equality +is not a record-preservation condition. Before-state records stay in size-prefixed FlatBuffer streams in +`--record-evidence-dir`; an index contains SHA-256 content-address digests and +stream offsets. Working indexes live in the container temporary filesystem; +completed baseline indexes are copied into the evidence directory. Comparison +uses a temporary copy so the baseline evidence stays unchanged. The after-state scan looks up every content address and +compares the complete bytes against that stream. Missing occurrences, different +bytes, extra occurrences of an existing address and unexplained additions fail. Counts remain +observations rather than the preservation criterion. + +Per-feed counts come from the node's own `/api/v1/stats` (`sources[]`, one row +per standard, provider, source and batch), read before and after from a stats +build that is not stale, was made after the node's startup writes and gives the +same counts twice. `feeds-before.json`, `feeds-after.json` and +`feed-counts-comparison.json` hold them. Every feed must keep its count; the only +differences allowed are `$ICN` feeds, whose total rise is at most the startup +persistence count (startup connector records can land in the local lane, which +stats does not list per feed), and an unchanged `$EPM` total when the local +profile refreshed. The per-standard counts must be equal apart from `$ICN`, which +must rise by exactly the startup persistence count. + +The only allowed additions are startup ICN records: each must name a connector +present before, have an update timestamp after the rollout began, and occur +once per connector. Their number must equal the current invocation's logged +startup persistence count. No additions are allowed when the old invocation +remains running. `ConnectorsHandler.persistLoop` calls `PersistConnectors` +after a 15-second boot delay; `BuildConnectorFrames` builds one record per +provider/source pair from stored summaries and the source ledger and assigns +a fresh `UPDATED_AT` timestamp. This accounts for startup records without +allowing arbitrary changes to existing records. + +The data API also merges the local node EPM into record reads. `epm.Service.Init` +rebuilds and persists that profile with a fresh signature timestamp. The +comparison recognizes only a Node EPM whose `/ipns/` address names the generated +test peer. After a service restart, one such profile may refresh its signature, +signature timestamp, and identity-attestation signatures and `issued_at` times. +`rebuildIdentityAttestationLocked` generates those attestations during startup. +Every other byte, including all profile fields, addresses, and public keys, must +match. Every new timestamp must fall between rollout start and the comparison. +Other EPM records require exact address and byte preservation. +Before-state frames and every new or changed after-state frame remain in +size-prefixed streams under the private evidence directory. A profile refresh +has its own result counter and does not count as an exact record match. + +Format-4 file hashes are taken at the script's explicit rehearsal barriers, +after the original service stops and after the bundle is restored but before +recovery starts that service. Only a newly added +`rollback-auxiliary.flatsqlmeta` is allowed; removed or changed files fail. +The conversion adds that snapshot immediately before activation. The barriers +are disabled unless the rehearsal explicitly supplies its pause directory. + +`results.json` summarizes case outcomes. Each case contains `driver.exit`, +`roll.exit`, `roll.log`, conversion attempt logs, `ledger.log`, aggregate `counts-*.json`, file hash lists, +`journal.log` and `result.json`. Peak migration RSS is sampled from `/proc`; +conversion duration is sampled between CONVERTING and SWITCHED ledger events. +All times describe Docker on the test machine. Archive digests are recorded in +`archives.json`. Test-generated identity material stays in its case directory; +do not publish it or copy it into a deployment. + +The harness and recovery script require execution on both source copies before +use on a production host. An image-build failure or incomplete case matrix +means the deployment gate has not passed. + +## Rehearsal findings + +The published v1.1.0 linux/arm64 archive fails to open the supplied format-4 +copies and creates format-5 marker bytes under `fsql4` with an empty store. +The two installed old binaries, verified against their supplied digests, each +reach readiness on their matching copied store in the amd64 rehearsal. + +The preceding v1.1.5 forward attempts stopped at the runtime check. A subsequent +study of that same verified linux/amd64 archive ran 60 self-tests while idle and +60 during a real conversion. All idle invocations passed. Two concurrent +invocations reported `interruptible_aot=false` and `patched=false`, with no +reported errors; the other 58 passed. Every concurrent invocation was bracketed +by a running converter. The source checks whether cancellation stops every +spawned executor within a short observation window. These observations establish +intermittent failures in this environment, without isolating their cause. The +reported AOT speed ratio does not decide the check's result. + +The verified v1.1.6 archive also exercised the early conversion retry: its first +attempt refused the runtime patches after three seconds, and its second entered +conversion. The guard is not bypassed and node code is unchanged. Each retry +requires an empty format-5 destination and shares the original hard deadline. + +The first input's post-switch case verifies 45 partitions and reaches SWITCHED +1,414 seconds after CONVERTING. At that point the retained format-4 tree, +including rollback material, contains 7,409,106,490 bytes and format 5 contains +11,421,381,390 bytes. The new unit's deliberate pre-start failure triggers +recovery to the old node, ready with the same running peer. A second invocation +is refused without restarting that node. These measurements describe emulated +Docker execution and are not a forecast for a deployment host. + +The preceding rehearsal's aggregate comparisons differed only in ICN counts: +33 additional records on the first input and 11 on the second. Startup journal +lines report exactly those persistence counts, including the precheck controls +that never stop or convert the store. Active-daemon file hashes cannot isolate +conversion changes. The record comparison and stopped-state barriers replace +those measurements; they require a completed rehearsal before they establish +preservation. The earlier failing results remain evidence of those earlier +measurements. + +The v1.1.6 rehearsal on both 2026-11-15 production store copies passed the +forward case with the restart check (host-01 copy with the linux/amd64 release: +4,419,593 records and 8,959 feeds preserved over the roll and over a clean stop +and start, no quarantine line; host-02 copy with the linux/arm64 build of the +same release as the new node, because its emulated amd64 conversion exceeded +180 minutes: every record of 13 standards and 670 feeds preserved, no quarantine +line) and the failed-conversion case (format-4 file hashes unchanged, every +record present after the return) on both copies. Earlier failing results +remain evidence of the measurements they made. diff --git a/deployment/scripts/roll-to-format5-rehearsal/case.py b/deployment/scripts/roll-to-format5-rehearsal/case.py new file mode 100644 index 0000000000..6ad7cb34c5 --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/case.py @@ -0,0 +1,463 @@ +#!/usr/bin/env python3 +"""Run inside an isolated container whose PID 1 is systemd.""" +import calendar +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import shutil +import subprocess +import sys +import time +import urllib.request + +import records + +ROOT = Path('/case') +UNIT = 'rehearsal-node.service' +READY = 'http://127.0.0.1:5001/api/v1/ready' +BIN = ROOT / 'base/sdn-bundle/bin/spacedatanetwork' + + +def command(*args, check=True): + return subprocess.run(args, capture_output=True, text=True, check=check).stdout.strip() + + +def wait_ready(seconds): + end = time.monotonic() + seconds + while time.monotonic() < end: + try: + with urllib.request.urlopen(READY, timeout=min(3, max(0.1, end-time.monotonic()))) as r: + if r.status == 200: + return + except Exception: + time.sleep(1) + raise RuntimeError('ready budget exceeded') + + +def hashes(): + result = {} + for p in sorted((ROOT / 'store/fsql4').rglob('*')): + if p.is_file(): + with p.open('rb') as f: + digest = hashlib.sha256() + for block in iter(lambda: f.read(1024 * 1024), b''): + digest.update(block) + result[str(p.relative_to(ROOT / 'store/fsql4'))] = digest.hexdigest() + return result + + +def stats_read(): + with urllib.request.urlopen('http://127.0.0.1:5001/api/v1/stats', timeout=60) as r: + return json.load(r) + + +def feeds(data): + # One row per feed: (standard, provider, source, batch) with its count. + return {'|'.join((f['schema'], f.get('provider_id', ''), f.get('source_name', ''), + f.get('batch_id', ''))): f['count'] for f in data.get('sources', [])} + + +def counts(name, after_unix): + # The stats lane rebuilds every few seconds; take two equal readings of a + # build that is not stale and was made after the node's startup writes. + end = time.monotonic() + 600 + previous = None + while time.monotonic() < end: + data = stats_read() + as_of = data.get('as_of') + fresh = not data.get('stale') and (as_of is None or + calendar.timegm(time.strptime(as_of, '%Y-%m-%dT%H:%M:%SZ')) > after_unix) + reading = ({s['schema']: s['count'] for s in data['schemas']}, feeds(data)) + if fresh and reading == previous: + break + previous = reading if fresh else None + time.sleep(6) + else: + raise RuntimeError('stats did not settle on a fresh build') + result, per_feed = reading + if not result or not sum(result.values()) or not per_feed: + raise RuntimeError('public stats has no populated per-standard or per-feed counts') + # Persist only aggregates, never source rows or records. + (ROOT / ('feeds-' + name + '.json')).write_text(json.dumps(per_feed, indent=1, sort_keys=True) + '\n') + return result, per_feed + + +def compare_feeds(before, after, boot_additions, profile_refreshes): + # Every feed keeps its count. The node's own startup writes are the only + # differences allowed: connector records ($ICN), whose feed rows rise by at + # most the persisted count (startup records can land in the local lane, + # which stats does not list per feed), and a refreshed local profile ($EPM), + # which replaces itself. The per-standard $ICN count must rise by exactly + # the persisted count; compare_standards checks that. + rows = {} + icn_delta = 0 + for key in sorted(set(before) | set(after)): + b, a = before.get(key), after.get(key) + if a == b: + continue + rows[key] = {'before': b, 'after': a} + if key.startswith('ICN.fbs|'): + icn_delta += (a or 0) - (b or 0) + unexplained = [k for k in rows if not k.startswith('ICN.fbs|')] + epm = [k for k in unexplained if k.startswith('EPM.fbs|')] + if profile_refreshes and epm and sum((rows[k]['after'] or 0) - (rows[k]['before'] or 0) for k in epm) == 0: + unexplained = [k for k in unexplained if k not in epm] + passed = not unexplained and 0 <= icn_delta <= boot_additions + return {'passed': passed, 'feeds_before': len(before), 'feeds_after': len(after), + 'feeds_equal': len(set(before) & set(after)) - len(rows), 'differences': rows, + 'icn_delta': icn_delta, 'startup_icn_persisted': boot_additions, + 'unexplained': unexplained} + + +def peer(): + invocation = command('systemctl', 'show', '-p', 'InvocationID', '--value', UNIT) + journal = command('journalctl', '--no-pager', '-o', 'cat', + '_SYSTEMD_INVOCATION_ID=' + invocation, '--grep=Peer ID: ') + matches = re.findall(r'Peer ID: ([a-zA-Z0-9]+)', journal) + if not matches: + raise RuntimeError('running node has no journal identity') + return matches[-1] + + +def boot_observation(): + invocation = command('systemctl', 'show', '-p', 'InvocationID', '--value', UNIT) + end = time.monotonic() + 180 + while time.monotonic() < end: + journal = command('journalctl', '--no-pager', '-o', 'cat', + '_SYSTEMD_INVOCATION_ID=' + invocation) + matches = re.findall(r'Connector lane: persisted (\d+) \$ICN row', journal) + if matches: + return invocation, sum(map(int, matches)) + time.sleep(1) + raise RuntimeError('connector startup persistence did not finish') + + +def compare_standards(before, after, boot_additions, profile_refreshes): + rows = {k: {'before': before.get(k), 'after': after.get(k)} + for k in sorted(set(before) | set(after)) if before.get(k) != after.get(k)} + icn = (after.get('ICN.fbs') or 0) - (before.get('ICN.fbs') or 0) + unexplained = [k for k in rows if k != 'ICN.fbs' and not (k == 'EPM.fbs' and profile_refreshes + and before.get(k) == after.get(k))] + return {'passed': not unexplained and icn == boot_additions, 'differences': rows, + 'icn_delta': icn, 'startup_icn_persisted': boot_additions, 'unexplained': unexplained} + + +def restart_check(seconds, expected, feeds_before, boot_ms, before_invocation, + first_invocation, first_additions): + # One clean stop and start of the new node on the converted store, then + # the same record and feed comparisons against the format-4 baseline. + stop_unix = time.time() + command('systemctl', 'stop', UNIT) + if command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) != '0': + raise RuntimeError('the new node did not stop') + command('systemctl', 'start', '--no-block', UNIT) + wait_ready(seconds) + if peer() != expected: + raise RuntimeError('identity changed over the restart') + invocation, additions = boot_observation() + if invocation == first_invocation: + raise RuntimeError('the node did not restart') + since = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(stop_unix - 1)) + journal = subprocess.run(['journalctl', '--no-pager', '-o', 'cat', '--utc', '--since', since, + '-u', UNIT], capture_output=True, text=True, check=True).stdout + corrupt = [line[:300] for line in journal.splitlines() + if 'quarantined' in line or 'P4_E_CORRUPT' in line] + (ROOT / 'restart-journal-corrupt-lines.json').write_text(json.dumps(corrupt, indent=1) + '\n') + after, feeds_after = counts('after-restart', time.time()) + (ROOT / 'counts-after-restart.json').write_text(json.dumps(after, sort_keys=True) + '\n') + preservation = records.compare('/evidence/records-before', after, boot_ms, + first_additions + additions, expected, True, starts=2, + additions_name='records-after-restart-additions') + (ROOT / 'record-preservation-after-restart.json').write_text(json.dumps(preservation, indent=2) + '\n') + refreshes = sum(row.get('refreshed_local_profiles', 0) for row in preservation['standards'].values()) + feed_result = compare_feeds(feeds_before, feeds_after, first_additions + additions, refreshes) + standard_result = compare_standards(json.loads((ROOT / 'counts-before.json').read_text()), after, + first_additions + additions, refreshes) + feed_result['standards'] = standard_result + feed_result['passed'] = feed_result['passed'] and standard_result['passed'] + (ROOT / 'feed-counts-comparison-after-restart.json').write_text(json.dumps(feed_result, indent=2) + '\n') + result = {'passed': not corrupt and preservation['passed'] and feed_result['passed'], + 'corrupt_journal_lines': len(corrupt), 'records_preserved': preservation['passed'], + 'feed_counts_preserved': feed_result['passed'], + 'startup_icn_persisted': [first_additions, additions]} + (ROOT / 'restart-check.json').write_text(json.dumps(result, indent=2) + '\n') + return result + + +def tree_size(path): + return sum(p.stat().st_size for p in path.rglob('*') if p.is_file()) if path.exists() else 0 + + +def run(case, minutes, seconds): + if Path('/proc/1/comm').read_text().strip() != 'systemd': + raise RuntimeError('PID 1 is not systemd') + with BIN.open('rb') as binary: + old_binary_sha256 = hashlib.file_digest(binary, 'sha256').hexdigest() + if case == 'disk': + # A real sparse loop filesystem bounds free space without filling a host disk. + source = ROOT / 'store-source' + (ROOT / 'store').rename(source) + (ROOT / 'store').mkdir() + image = ROOT / 'disk.img' + size = tree_size(source) + 2 * 1024**3 + command('truncate', '-s', str(size), str(image)) + command('mkfs.ext4', '-q', '-F', '-m', '0', str(image)) + command('mount', '-o', 'loop', str(image), str(ROOT / 'store')) + shutil.copytree(source, ROOT / 'store', dirs_exist_ok=True) + (ROOT / 'disk-filesystem.json').write_text(json.dumps({ + 'filesystem': 'ext4', 'capacity_bytes': size}) + '\n') + (ROOT / 'home').mkdir(exist_ok=True) + password = ROOT / 'test-password' + if not password.exists(): + password.write_text(os.urandom(32).hex()) + password.chmod(0o600) + os.environ['SDN_KEY_PASSWORD_FILE'] = str(password) + os.environ['XDG_CACHE_HOME'] = '/engine-cache' + os.environ['HOME'] = '/case/home' + # Empty bootstrap lists trigger built-in public defaults in these releases. + # A synthetic identity at an unused loopback port prevents that fallback. + alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + value = int.from_bytes(bytes.fromhex('002408011220') + bytes([1]) * 32, 'big') + encoded = '' + while value: + value, digit = divmod(value, 58) + encoded = alphabet[digit] + encoded + isolated_peer = '1' + encoded + # All keys belong to this isolated test. The container has no external network. + (ROOT / 'config.yaml').write_text('''mode: full +flows: + services: [] +network: + listen: [/ip4/127.0.0.1/tcp/4001] + bootstrap: [/ip4/127.0.0.1/tcp/9/p2p/ISOLATED_PEER] + edge_relays: [] + enable_relay: never +storage: + path: /case/store + max_size: 200GB + gc_interval: 24h +peers: + trusted_peers: [] + registry_path: /case/empty-peer-registry.json + enable_dht: false + enable_mdns: false +tor: + enabled: false +admin: + enabled: true + listen_addr: 127.0.0.1:5001 + tls_mode: disabled + require_auth: false +setup: + data_path: /case/store +'''.replace('ISOLATED_PEER', isolated_peer)) + unit = Path('/etc/systemd/system') / UNIT + unit.write_text('''[Unit] +Description=Isolated store rehearsal +[Service] +Type=simple +User=root +Environment=HOME=/case/home XDG_CACHE_HOME=/engine-cache SDN_KEY_PASSWORD_FILE=/case/test-password +ExecStart=/case/base/sdn-bundle/bin/spacedatanetwork daemon --config /case/config.yaml +Restart=always +RestartSec=15 +TimeoutStopSec=75 +''') + command('systemctl', 'daemon-reload') + command('systemctl', 'start', '--no-block', UNIT) + wait_ready(seconds) + command('systemctl', 'is-active', '--quiet', UNIT) + (ROOT / 'baseline.json').write_text(json.dumps({'ready': True, 'http_status': 200}) + '\n') + expected = peer() + before_invocation, _ = boot_observation() + before, feeds_before = counts('before', time.time()) + (ROOT / 'counts-before.json').write_text(json.dumps(before, sort_keys=True) + '\n') + records.snapshot('/evidence/records-before', before) + before_hashes = None + stopped_hash_result = None + pause = ROOT / 'pause' + pause.mkdir() + # Populate each release's engine cache before the outage. + with open(ROOT / 'prewarm.log', 'w') as log: + subprocess.run(['/case/base/staged/bin/spacedatanetwork', 'prewarm-aot', '--config', + '/case/config.yaml'], stdout=log, stderr=subprocess.STDOUT, check=True) + if case == 'after-switch': + drop = Path('/etc/systemd/system') / (UNIT + '.d') + drop.mkdir() + (drop / 'fail-new.conf').write_text('''[Service] +ExecStartPre=/bin/sh -c 'test ! -d /case/store/fsql5' +''') + command('systemctl', 'daemon-reload') + if case == 'disk': + # Verify that the actual filesystem fails the production space formula. + vfs = os.statvfs(ROOT / 'store') + available = vfs.f_bavail * vfs.f_frsize + required = 3 * tree_size(ROOT / 'store/fsql4') + (ROOT / 'store/auxiliary.flatsqlmeta').stat().st_size + 1073741824 + 5000000000 + (ROOT / 'disk.json').write_text(json.dumps({'available_bytes': available, 'required_bytes': required}) + '\n') + if available >= required: + return {'status': 'not run', 'reason': 'work filesystem has enough free space; supply a constrained filesystem'} + args = ['/bin/bash', '/case/roll.sh', UNIT, '/case/base', '/case/base/staged', + '/case/config.yaml', '/case/store', READY, + 'WrongExpectedPeer' if case == 'wrong-peer' else expected, + '/case/ledger.log', 'rehearsal', case, '1' if case == 'budget' else str(minutes), + str(seconds), '/case/home'] + baseline_pid = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) + start = time.monotonic() + boot_ms = int(time.time() * 1000) + proc = subprocess.Popen(['systemd-run', '--unit=rehearsal-roll', '--collect', '--wait', '--pipe', + '--setenv=SDN_ROLL_REHEARSAL_PAUSE_DIR=/case/pause', *args], + stdout=open(ROOT / 'roll.log', 'w'), stderr=subprocess.STDOUT) + injected = False + peak_kib = 0 + migration_start = None + migration_end = None + switched_bytes = None + while proc.poll() is None: + for point in ('stopped', 'restored'): + if (pause / (point + '.waiting')).exists() and not (pause / (point + '.continue')).exists(): + if command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) != '0': + raise RuntimeError('hash barrier has a running node') + current = hashes() + (ROOT / ('hashes-' + point + '.json')).write_text(json.dumps(current, sort_keys=True) + '\n') + if point == 'stopped': + before_hashes = current + elif case != 'after-switch': + added = set(current) - set(before_hashes) + missing = set(before_hashes) - set(current) + changed = [key for key in set(before_hashes) & set(current) + if before_hashes[key] != current[key]] + stopped_hash_result = {'passed': not missing and not changed and + added <= {'rollback-auxiliary.flatsqlmeta'}, + 'missing': len(missing), 'changed': len(changed), + 'added': sorted(added)} + (ROOT / 'stopped-hash-comparison.json').write_text(json.dumps(stopped_hash_result) + '\n') + (pause / (point + '.continue')).touch() + ledger = (ROOT / 'ledger.log').read_text() if (ROOT / 'ledger.log').exists() else '' + matches = re.findall(r' CONVERTING .*pid=(\d+)', ledger) + match = matches[-1] if matches else None + if match: + pid = int(match) + migration_start = migration_start or time.monotonic() + stat = Path('/proc') / str(pid) / 'status' + if stat.exists(): + rss = re.search(r'^VmRSS:\s+(\d+)', stat.read_text(), re.M) + if rss: + peak_kib = max(peak_kib, int(rss[1])) + pending = ROOT / 'store/.store-format5-pending' + if not injected and pending.exists() and time.monotonic() - migration_start >= 10: + if case in ('interrupted', 'script-term', 'script-hup'): + if case == 'interrupted': + command_line = (Path('/proc') / str(pid) / 'cmdline').read_bytes().split(b'\0') + if b'store-migrate' not in command_line: + raise RuntimeError('conversion PID no longer matches') + os.kill(pid, signal.SIGTERM) + else: + script_pid = int(command('systemctl', 'show', '-p', 'MainPID', '--value', 'rehearsal-roll')) + command_line = (Path('/proc') / str(script_pid) / 'cmdline').read_bytes() + if b'/case/roll.sh' not in command_line: + raise RuntimeError('script PID no longer matches') + os.kill(script_pid, signal.SIGTERM if case == 'script-term' else signal.SIGHUP) + (ROOT / 'injected.json').write_text(json.dumps({'case': case, 'conversion_pid': pid, 'seconds': time.monotonic()-start}) + '\n') + injected = True + if ' SWITCHED ' in ledger and migration_end is None: + migration_end = time.monotonic() + switched_bytes = tree_size(ROOT / 'store/fsql5') + time.sleep(0.25) + code = proc.wait() + (ROOT / 'roll.exit').write_text(str(code) + '\n') + (ROOT / 'conversion-observation.json').write_text(json.dumps({ + 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, + 'peak_conversion_rss_kib_sampled': peak_kib, + 'switched_fsql5_bytes': switched_bytes}) + '\n') + ledger = (ROOT / 'ledger.log').read_text() + wait_ready(seconds) + if peer() != expected: + raise RuntimeError('identity changed') + if case != 'forward': + with BIN.open('rb') as binary: + selected_sha256 = hashlib.file_digest(binary, 'sha256').hexdigest() + (ROOT / 'bundle-selection.json').write_text(json.dumps({ + 'old_binary_selected': selected_sha256 == old_binary_sha256, + 'expected_sha256': old_binary_sha256, 'selected_sha256': selected_sha256}) + '\n') + if selected_sha256 != old_binary_sha256: + raise RuntimeError('the selected bundle is not the original binary') + after_invocation, boot_additions = boot_observation() + after, feeds_after = counts('after', time.time()) + (ROOT / 'counts-after.json').write_text(json.dumps(after, sort_keys=True) + '\n') + preservation = records.compare('/evidence/records-before', after, boot_ms, + boot_additions if after_invocation != before_invocation else 0, + expected, after_invocation != before_invocation) + (ROOT / 'record-preservation.json').write_text(json.dumps(preservation, indent=2) + '\n') + refreshes = sum(row.get('refreshed_local_profiles', 0) for row in preservation['standards'].values()) + feed_result = compare_feeds(feeds_before, feeds_after, + boot_additions if after_invocation != before_invocation else 0, refreshes) + standard_result = compare_standards(before, after, + boot_additions if after_invocation != before_invocation else 0, refreshes) + feed_result['standards'] = standard_result + feed_result['passed'] = feed_result['passed'] and standard_result['passed'] + (ROOT / 'feed-counts-comparison.json').write_text(json.dumps(feed_result, indent=2) + '\n') + (ROOT / 'observations.json').write_text(json.dumps({ + 'ready': True, 'running_peer_unchanged': True, 'counts_equal': before == after, + 'roll_exit': code, 'injected': injected, + 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, + 'peak_conversion_rss_kib_sampled': peak_kib, 'switched_fsql5_bytes': switched_bytes, + 'stopped_format4_hashes': stopped_hash_result, 'records_preserved': preservation['passed'], + 'feed_counts_preserved': feed_result['passed']}) + '\n') + if any(' ' + event + ' ' in ledger for event in + ('SUCCESS', 'ROLLED_BACK_BEFORE_SWITCH', 'ROLLED_BACK_AFTER_SWITCH')): + replay_pid = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) + with open(ROOT / 'second-run.log', 'w') as log: + second = subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode + (ROOT / 'second-run.exit').write_text(str(second) + '\n') + unchanged = command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) == replay_pid + (ROOT / 'second-run.json').write_text(json.dumps({ + 'refused': second != 0, 'daemon_pid_unchanged': unchanged}) + '\n') + if second == 0 or not unchanged: + raise RuntimeError('second run was not refused without restarting the node') + restart_result = None + if case == 'forward' and code == 0 and ' SUCCESS ' in ledger: + restart_result = restart_check(seconds, expected, feeds_before, boot_ms, + before_invocation, after_invocation, boot_additions) + if not preservation['passed']: + raise RuntimeError('record preservation failed') + if not feed_result['passed']: + raise RuntimeError('per-feed counts changed') + if restart_result is not None and not restart_result['passed']: + raise RuntimeError('the converted store changed over a clean stop and start') + if case == 'forward': + if code != 0 or ' SUCCESS ' not in ledger: + raise RuntimeError('forward conversion failed') + elif case in ('wrong-peer', 'disk'): + if code == 0 or ' STOPPED ' in ledger or ' PRECHECK_FAILED ' not in ledger: + raise RuntimeError('precheck did not refuse before stop') + if command('systemctl', 'show', '-p', 'MainPID', '--value', UNIT) != baseline_pid: + raise RuntimeError('precheck changed the running daemon') + if not (ROOT / 'base/staged').exists() or (ROOT / 'store/fsql5').exists(): + raise RuntimeError('precheck changed bundle or store selection') + else: + event = 'ROLLED_BACK_AFTER_SWITCH' if case == 'after-switch' else 'ROLLED_BACK_BEFORE_SWITCH' + if code == 0 or (' ' + event + ' ') not in ledger: + raise RuntimeError('expected automatic rollback did not complete') + if case in ('interrupted', 'script-term', 'script-hup') and not injected: + raise RuntimeError('failure was not injected during conversion') + if case == 'budget' and 'reason=conversion_budget' not in ledger: + raise RuntimeError('conversion did not reach its budget') + if case != 'after-switch' and not (stopped_hash_result or {}).get('passed'): + raise RuntimeError('stopped format-4 file hashes changed or were not measured') + return {'status': 'passed', 'roll_exit': code, 'seconds': round(time.monotonic()-start, 2), + 'conversion_seconds': round(migration_end-migration_start, 2) if migration_end else None, + 'peak_conversion_rss_kib_sampled': peak_kib, + 'fsql4_bytes': tree_size(ROOT / 'store/fsql4'), + 'fsql5_bytes': tree_size(ROOT / 'store/fsql5'), 'counts': after} + + +if __name__ == '__main__': + try: + result = run(sys.argv[1], int(sys.argv[2]), int(sys.argv[3])) + except Exception as error: + result = {'status': 'failed', 'reason': str(error)} + (ROOT / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + sys.exit(0 if result['status'] == 'passed' else 1) diff --git a/deployment/scripts/roll-to-format5-rehearsal/records.py b/deployment/scripts/roll-to-format5-rehearsal/records.py new file mode 100644 index 0000000000..37a59c0623 --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/records.py @@ -0,0 +1,327 @@ +"""Compare every scanned record's content address and exact stored bytes. + +Raw evidence stays in size-prefixed FlatBuffer streams. The SQLite index holds +only digests and stream offsets. Public results contain per-standard totals. +""" +import hashlib +import json +import re +from pathlib import Path +import sqlite3 +import shutil +import tempfile +import struct +import time +import urllib.request + +API = 'http://127.0.0.1:5001/api/v1/data/scan' +MEDIA = 'application/vnd.sdn.flatbuffers.stream' + + +def scalar(data, offset, fmt): + return struct.unpack_from('<' + fmt, data, offset)[0] + + +def field(data, slot, table=None): + if table is None: + table = scalar(data, 0, 'I') + vtable = table - scalar(data, table, 'i') + entry = 4 + slot * 2 + if entry >= scalar(data, vtable, 'H'): + return 0 + offset = scalar(data, vtable + entry, 'H') + return table + offset if offset else 0 + + +def number(data, slot, fmt): + offset = field(data, slot) + return scalar(data, offset, fmt) if offset else 0 + + +def string(data, slot): + offset = field(data, slot) + if not offset: + return b'' + start = offset + scalar(data, offset, 'I') + size = scalar(data, start, 'I') + return data[start + 4:start + 4 + size] + + +def local_profile(data, peer): + # EPM slots: MULTIFORMAT_ADDRESS=14, SIGNATURE=15, + # SIGNATURE_TIMESTAMP=16, ENTITY_TYPE=18 (Node=1). + prefix = b'' + if data[4:8] != b'$EPM' and data[8:12] == b'$EPM': + prefix, data = data[:4], data[4:] + if not peer or data[4:8] != b'$EPM' or number(data, 18, 'b') != 1: + return None + address = field(data, 14) + signature = field(data, 15) + timestamp = field(data, 16) + if not address or not signature or not timestamp: + return None + vector = address + scalar(data, address, 'I') + addresses = [] + for index in range(scalar(data, vector, 'I')): + offset = vector + 4 + index * 4 + start = offset + scalar(data, offset, 'I') + size = scalar(data, start, 'I') + addresses.append(data[start + 4:start + 4 + size]) + identity = peer.encode() + if b'/ipns/' + identity not in addresses: + return None + start = signature + scalar(data, signature, 'I') + size = scalar(data, start, 'I') + value = data[start + 4:start + 4 + size] + if not value or any(byte not in b'0123456789abcdefABCDEF' for byte in value): + return None + masked = bytearray(data) + masked[start + 4:start + 4 + size] = b'\0' * size + masked[timestamp:timestamp + 8] = b'\0' * 8 + issued_times = [] + proofs = field(data, 17) + if proofs: + vector = proofs + scalar(data, proofs, 'I') + for index in range(scalar(data, vector, 'I')): + entry = vector + 4 + index * 4 + table = entry + scalar(data, entry, 'I') + # ChainProof slots: SIGNATURE=4, SIGNED_PAYLOAD=5. + signature = field(data, 4, table) + payload = field(data, 5, table) + if not signature or not payload: + return None + start = signature + scalar(data, signature, 'I') + size = scalar(data, start, 'I') + if not size: + return None + masked[start + 4:start + 4 + size] = b'\0' * size + start = payload + scalar(data, payload, 'I') + size = scalar(data, start, 'I') + try: + decoded = bytes.fromhex(data[start + 4:start + 4 + size].decode('ascii')) + document = json.loads(decoded) + except (ValueError, UnicodeError): + return None + matches = list(re.finditer(rb'"issued_at":([0-9]+)', decoded)) + if len(matches) != 1 or document.get('issued_at') != int(matches[0][1]): + return None + issued_times.append(document['issued_at']) + first, last = matches[0].span(1) + masked[start + 4 + first * 2:start + 4 + last * 2] = b'0' * ((last - first) * 2) + return prefix + bytes(masked), scalar(data, timestamp, 'q'), tuple(issued_times) + + +def request_frame(schema, cursor): + # QRP slots: KIND=0 (Request, the default), SCHEMA_NAME=5, + # LIMIT=20, CURSOR=21. A forward-built FlatBuffer needs no dependency. + vtable = 8 + table = 56 + data = bytearray(72) + struct.pack_into(' 1: + stream.seek(prior[0] + 4) + if prior[1] != len(data) or stream.read(prior[1]) != data: + raise RuntimeError('one content address has different baseline bytes') + stream.seek(offset) + stream.write(struct.pack('= occurrences: + unexplained += 1 + db.execute('UPDATE records SET seen=seen+1 WHERE cid=?', (cid,)) + matched += 1 + else: + added += 1 + after_stream.write(struct.pack('= starts: + unexplained += 1 + added_connectors[connector] = added_connectors.get(connector, 0) + 1 + elif schema == 'EPM.fbs' and len(profiles) == 1 and not refreshed_profiles: + profile = local_profile(data, local_peer) + old_cid, old_profile = profiles[0] + unseen = db.execute('SELECT seen FROM records WHERE cid=?', (old_cid,)).fetchone()[0] == 0 + if profile and unseen and profile[0] == old_profile[0] and \ + all(stamp <= boot_ms // 1000 for stamp in (old_profile[1], *old_profile[2])) and \ + all(boot_ms // 1000 <= stamp <= int(time.time()) for stamp in (profile[1], *profile[2])): + db.execute('UPDATE records SET seen=seen+1 WHERE cid=?', (old_cid,)) + refreshed_profiles += 1 + else: + unexplained += 1 + else: + unexplained += 1 + if (matched + added) % 20000 == 0: + db.commit() + missing = db.execute('SELECT coalesce(sum(max(occurrences-seen,0)),0) FROM records').fetchone()[0] + distinct_before = db.execute('SELECT count(*) FROM records').fetchone()[0] + db.commit() + db.close() + temporary.cleanup() + if schema == 'ICN.fbs' and added != expected_boot_additions: + unexplained += abs(added - expected_boot_additions) + results[schema] = dict(before=before[schema], distinct_before=distinct_before, + baseline_duplicate_occurrences=before[schema]-distinct_before, + matched=matched, added=added, + refreshed_local_profiles=refreshed_profiles, + missing=missing, changed=changed, unexplained=unexplained) + passed = all(not any(row.get(key, 0) for key in + ('missing', 'changed', 'unexplained', 'unexpected_standard')) + for row in results.values()) + return {'passed': passed, 'standards': results, + 'method': 'SHA-256 content address lookup and exact bytes; one generated local profile may refresh only signatures and their timestamps'} diff --git a/deployment/scripts/roll-to-format5-rehearsal/run.py b/deployment/scripts/roll-to-format5-rehearsal/run.py new file mode 100755 index 0000000000..006e623ae1 --- /dev/null +++ b/deployment/scripts/roll-to-format5-rehearsal/run.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Isolated systemd rehearsal. Inputs and retained evidence are explicit paths.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile + +CASES = ('forward', 'interrupted', 'budget', 'after-switch', 'wrong-peer', + 'disk', 'script-term', 'script-hup') + + +def sha256(path): + digest = hashlib.sha256() + with open(path, 'rb') as stream: + for block in iter(lambda: stream.read(1 << 20), b''): + digest.update(block) + return digest.hexdigest() + + +def verify(archive, checksums): + digest = sha256(archive) + expected = [line.split()[0] for line in checksums.read_text().splitlines() + if len(line.split()) == 2 and line.split()[1].lstrip('*') == archive.name] + if expected != [digest]: + raise ValueError('release checksum mismatch: ' + archive.name) + return {'archive': archive.name, 'sha256': digest} + + +def unpack(archive, destination): + destination.mkdir(parents=True) + with tarfile.open(archive) as bundle: + members = bundle.getmembers() + tops = {m.name.split('/')[0] for m in members if m.name} + if len(tops) != 1: + raise ValueError('archive needs one top-level bundle directory') + for member in members: + if member.name.startswith('/') or '..' in member.name.split('/') or \ + not (member.isfile() or member.isdir() or member.issym()): + raise ValueError('archive member outside the bundle: ' + member.name) + unpacked = destination.parent / (destination.name + '.unpack') + unpacked.mkdir() + subprocess.run(['tar', '-xzf', str(archive), '--no-same-owner', '-C', str(unpacked)], check=True) + source = destination.parent / (destination.name + '.unpack') / tops.pop() + for child in source.iterdir(): + shutil.move(str(child), destination / child.name) + source.rmdir() + source.parent.rmdir() + if not (destination / 'bin/spacedatanetwork').is_file(): + raise ValueError('archive has no bundle launcher') + + +def private(name): + return name in ('keys', 'secrets', 'auth.db') or 'udl' in name.lower() + + +def copy_store(source, destination): + # A copy-on-write clone where the filesystem supports one, then remove + # private material, which never takes part in a rehearsal. + clone = ['cp', '-c', '-p', '-R', str(source), str(destination)] if sys.platform == 'darwin' \ + else ['cp', '-a', '--reflink=auto', str(source), str(destination)] + subprocess.run(clone, check=True) + for directory, names, files in os.walk(destination, topdown=True): + for name in [n for n in names if private(n)]: + shutil.rmtree(os.path.join(directory, name)) + names.remove(name) + for name in files: + if private(name): + os.unlink(os.path.join(directory, name)) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + for name in ('new-archive', 'new-checksums', + 'work-dir', 'record-evidence-dir'): + ap.add_argument('--' + name, required=True, type=Path) + ap.add_argument('--cache-dir', type=Path, help='shared engine cache; contains no identities') + ap.add_argument('--store', required=True, action='append', type=Path) + ap.add_argument('--old-binary', required=True, action='append', type=Path) + ap.add_argument('--old-sha256', required=True, action='append') + ap.add_argument('--arch', choices=('arm64', 'amd64'), default='amd64') + ap.add_argument('--case', choices=CASES, action='append') + ap.add_argument('--conversion-minutes', type=int, default=90) + ap.add_argument('--ready-seconds', type=int, default=300) + ap.add_argument('--cpus', default='2') + ap.add_argument('--memory', default='6g') + args = ap.parse_args() + if len(args.old_binary) != len(args.store) or len(args.old_sha256) != len(args.store): + ap.error('each store requires its matching old binary and SHA-256') + here = Path(__file__).resolve().parent + work = args.work_dir.resolve() + work.mkdir(parents=True, exist_ok=False) + cache = (args.cache_dir or (work / 'cache')).resolve() + cache.mkdir(parents=True, exist_ok=True) + results = [] + active = None + image = 'sdn-format5-rehearsal:' + str(os.getpid()) + try: + verified = [verify(args.new_archive, args.new_checksums)] + for binary, expected in zip(args.old_binary, args.old_sha256): + digest = sha256(binary) + if digest != expected: + raise ValueError('old binary checksum mismatch') + verified.append({'old_binary': binary.name, 'sha256': digest}) + (work / 'archives.json').write_text(json.dumps(verified, indent=2) + '\n') + context = work / 'image' + context.mkdir(exist_ok=False) + shutil.copy2(here / 'Dockerfile', context / 'Dockerfile') + with open(work / 'image.log', 'w') as log: + subprocess.run(['docker', 'build', '--platform=linux/' + args.arch, + '-t', image, str(context)], stdout=log, stderr=subprocess.STDOUT, check=True) + for number, source in enumerate(args.store, 1): + for case in args.case or CASES: + directory = work / ('store-' + str(number)) / case + directory.mkdir(parents=True) + copy_store(source, directory / 'store') + old_bundle = directory / 'base/sdn-bundle' + (old_bundle / 'bin').mkdir(parents=True) + old_binary = args.old_binary[number - 1] + shutil.copy2(old_binary, old_bundle / 'bin/spacedatanetwork') + shutil.copy2(old_binary.parent / 'manifest.json', old_bundle / 'manifest.json') + unpack(args.new_archive, directory / 'base/staged') + shutil.copy2(here.parent / 'roll-to-format5.sh', directory / 'roll.sh') + shutil.copy2(here / 'case.py', directory / 'case.py') + shutil.copy2(here / 'records.py', directory / 'records.py') + evidence = args.record_evidence_dir.resolve() / ('store-' + str(number)) / case + evidence.mkdir(parents=True, exist_ok=False) + active = 'sdn-format5-' + str(os.getpid()) + '-' + str(number) + '-' + case + with open(directory / 'container.log', 'w') as log: + subprocess.run(['docker', 'run', '-d', '-t', '--name', active, '--hostname=sdn-rehearsal', '--privileged', + '--cgroupns=host', '--network=none', '--platform=linux/' + args.arch, + '--cpus=' + args.cpus, '--memory=' + args.memory, '--tmpfs=/run', '--tmpfs=/run/lock', + '--tmpfs=/tmp', '-v', '/sys/fs/cgroup:/sys/fs/cgroup:rw', + '-v', str(cache) + ':/engine-cache', + '-v', str(evidence) + ':/evidence', + '-v', str(directory) + ':/case', image], + stdout=log, stderr=subprocess.STDOUT, check=True) + try: + with open(directory / 'driver.log', 'w') as log: + status = subprocess.run(['docker', 'exec', active, 'python3', '/case/case.py', + case, str(args.conversion_minutes), str(args.ready_seconds)], + stdout=log, stderr=subprocess.STDOUT).returncode + (directory / 'driver.exit').write_text(str(status) + '\n') + result_file = directory / 'result.json' + result = json.loads(result_file.read_text()) if result_file.exists() else { + 'status': 'failed', 'reason': 'driver exited without a result'} + results.append({'store': number, 'case': case, **result}) + finally: + with open(directory / 'journal.log', 'w') as log: + subprocess.run(['docker', 'exec', active, 'journalctl', '--no-pager', '-o', 'short-iso'], + stdout=log, stderr=subprocess.STDOUT) + # Docker addresses only the container created above. + subprocess.run(['docker', 'stop', active], check=True, stdout=subprocess.DEVNULL) + subprocess.run(['docker', 'rm', active], check=True, stdout=subprocess.DEVNULL) + active = None + if not (directory / 'driver.exit').exists() or not (directory / 'result.json').exists(): + raise RuntimeError('incomplete case retained; inspect before creating another store') + # Retain evidence, then reclaim this completed case before another copy. + for name in ('store', 'store-source', 'base'): + if (directory / name).exists(): + shutil.rmtree(directory / name) + for backup in directory.glob('store.roll-backup-*'): + for log in backup.glob('conversion-attempt-*.log'): + shutil.copy2(log, directory / log.name) + shutil.rmtree(backup) + (directory / 'disk.img').unlink(missing_ok=True) + (work / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + except Exception as error: + (work / 'blocked.txt').write_text(str(error) + '\n') + raise + finally: + # Logs and record evidence survive cleanup; interrupted cases remain inspectable. + if active: + subprocess.run(['docker', 'stop', active], stdout=subprocess.DEVNULL) + (work / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + return int(any(r['status'] != 'passed' for r in results)) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/deployment/scripts/roll-to-format5.md b/deployment/scripts/roll-to-format5.md new file mode 100644 index 0000000000..711a31422d --- /dev/null +++ b/deployment/scripts/roll-to-format5.md @@ -0,0 +1,195 @@ +# Roll a stopped format-4 store to format 5 + +`roll-to-format5.sh` replaces a complete bundle, converts the store, and checks +readiness and identity. A failure after stopping the service triggers automatic +recovery. The script requires Linux, Bash, GNU coreutils, systemd and curl, and +runs as root. The service also runs as root. No schema or record data is printed +by the script. Conversion output comes from the node binary. + +The release-source recovery contract is verified against v1.1.5 and v1.1.6. End-to-end +validation on production-store copies is required before deployment. The +included rehearsal installs systemd from official package archives; syntax checks do not +establish recovery safety. + +## Preparation + +1. Acquire the deployment lock with the deployment's exclusive lock procedure. + Record the holder, token and manual-install reason in the deployment ledger. + Keep that lock through recovery or success. The script records the supplied + holder and token; it does not acquire or release an external lock. +2. Download the complete release archive, verify it against the release checksum + file, and unpack it without preserving archive ownership. Verify the manifest, + platform, version and trust roots. Stage it on the same filesystem as the + bundle base. The live folder is named `sdn-bundle`. The store is outside both + bundle directories. Neither bundle is an ancestor of the other. +3. Check the service configuration and store root agree. Check any module catalog + hashes required by the deployment. Run `version`, `show-identity -c CONFIG`, + and `key fingerprint-check` with the service's environment. Never add + `--show-mnemonic`. Record the running daemon's peer ID. +4. Run the staged binary's `prewarm-aot --config CONFIG` as root with the + service's environment and cache HOME. Record the old binary checksum. +5. Ensure the ledger is writable and private. Ensure the ready URL is the node's + unauthenticated ready endpoint and is usable by both releases. A local URL + avoids a proxy's cached response. Ensure the current invocation's journal + contains the node's `Peer ID:` startup line. + +The script reads `systemctl show -p Environment --value` without printing it. +It parses quoted words without evaluating shell code. Unsupported escape +sequences, non-root services and `EnvironmentFile` settings are refused before +the outage. Supply the service's HOME as the final argument when it differs +from the default. Explicit HOME takes precedence over the unit's Environment. +The service's PATH is honored when present; the default is the system PATH. +Service namespaces and filesystem isolation need equivalent command access: +configuration and store paths must identify the same files outside the unit. + +The disk gate requires free bytes of at least: + +``` +3 * du_bytes(fsql4) + bytes(auxiliary.flatsqlmeta) + 1073741824 + 5000000000 +``` + +An auxiliary metadata file is required because this procedure promises its +recovery snapshot. Symbolic links for the live bundle, fsql4 and auxiliary file +are refused. Check all paths and budgets before the outage. + +## Invocation + +Set the variables to the deployment's values, then start one transient service: + +```bash +systemd-run --unit="$ROLL_UNIT" --collect \ + /bin/bash "$SCRIPT" \ + "$NODE_UNIT" "$BUNDLE_BASE" "$STAGED_BUNDLE" "$CONFIG" "$STORE_ROOT" \ + "$READY_URL" "$EXPECTED_PEER" "$LEDGER" "$HOLDER" "$TOKEN" \ + "$CONVERSION_MINUTES" "$READY_SECONDS" "$NODE_HOME" +journalctl -u "$ROLL_UNIT" -f +``` + +The first budget is a positive whole number of minutes; the second is a positive +whole number of seconds. The readiness budget applies separately to the new +node and to recovery. Stop waits are bounded at 90 seconds. Conversion is +supervised as a child of the script. Each attempt writes a log in the side-file +backup directory; CONVERTING names it. Completed attempt output is also printed. +At the conversion deadline the script kills and waits for that exact child +before inspecting the switch. It does not leave an independent migration job. +The conversion command includes `--store STORE_ROOT` as well as `--config` to +bind the conversion to the checked root. + +SIGTERM, SIGHUP and SIGINT sent to the script invoke the same recovery path. +SIGKILL, power loss, a kernel failure or the loss of the filesystem prevents +shell traps from running. The transient service detaches the outage from the +operator's login session. Do not stop the entire transient unit to simulate a +signal to the script: systemd can also terminate its children. + +## Ledger + +Every event is appended as one UTC line and also written to standard output: + +``` + token= +``` + +| Event | Meaning | +| --- | --- | +| PRECHECK | Read-only checks started, apart from the ledger and the identity command's permission enforcement. | +| DISK | Required and available byte counts. | +| RUNTIME_CHECK | `attempt=N`: the staged binary runs `substrate-selftest --require-patched` with the service environment. | +| RUNTIME_CHECK_OK | The staged runtime passed all required checks. | +| RUNTIME_CHECK_REFUSED | This precheck attempt failed. | +| RUNTIME_CHECK_FAILED | All five precheck attempts failed; the service remains running. | +| PRECHECK_OK | Environment, identity, path, space and runtime checks passed. | +| REHEARSAL_PAUSED | An explicitly enabled rehearsal barrier awaits acknowledgment. | +| STOPPED | MainPID is zero and the unit is inactive or failed. | +| SWAPPED | The old bundle is retained and the staged bundle occupies the live name. | +| BACKED_UP | Stopped-state side files are copied beside the store. | +| CONVERTING | Exact conversion PID and time budget. | +| CONVERSION_RETRY | An early runtime-patches refusal left no format-5 content; another attempt follows. | +| SWITCHED | Format bytes are `05 00`; fsql4 and its auxiliary snapshot exist. | +| SUCCESS | The new service answers ready and its invocation journal has the expected peer ID. | +| FAILED | A checked step failed; recovery follows if stopping began. | +| PRECHECK_FAILED | No outage began. | +| ROLLBACK | Recovery started; `after_switch` records whether fsql5 exists. | +| ROLLED_BACK_BEFORE_SWITCH | The original bundle is serving format 4 with the expected identity. | +| ROLLED_BACK_AFTER_SWITCH | Format 5 is retained under a failed name, side files and auxiliary metadata are restored, and the original bundle is serving. | +| ROLLBACK_FAILED | Recovery stopped at the named step. Human intervention is required. | + +Argument and path-resolution errors can exit before ledger initialization. +Success exits zero. Refusal, conversion failure and completed recovery exit +nonzero. A failed recovery also exits nonzero. + +## Recovery artifacts + +The original bundle becomes `sdn-bundle.old--`. Recovery moves the +replacement to `sdn-bundle.failed--` and restores the original live +name. Side files are copied into `STORE_ROOT.roll-backup--`: +`dataset-catalogs`, `dataset-feed-head-sync`, `control.flatsqldb`, +`fsql4-migrate.json`, `source-metrics.db`, `storefront.db`, `peers.db`, +`trusted-import-lanes.json` and `homepage.json`, when present. SQLite `-wal`, +`-shm` and `-journal` companions of the three `.db` files are included. +`keys`, `secrets` and `auth.db` are excluded. + +After a switch, recovery moves `fsql5` to `fsql5.failed--` and copies +`fsql4/rollback-auxiliary.flatsqlmeta` over the root auxiliary metadata. Side +files from the failed run are moved to the backup's `changed` folder before +restoring their originals, including original absence. No retained store or +bundle is deleted. Records accepted after conversion are absent from the old +store. Quiesce external ingestion during the deployment window. + +The v1.1.5 converter removes its unpublished candidate when returning an error; +an abruptly terminated converter can leave `.store-format5-pending`. A subsequent +conversion reclaims only a candidate whose ownership marker matches the source. +The script leaves this candidate to the converter. A termination immediately +before the atomic switch can leave the auxiliary rollback snapshot in fsql4. + +A repeat with the original arguments is refused: after success fsql5 exists and +the staged directory is consumed; after recovery the staged directory is also +consumed. A deliberate retry requires a newly staged bundle and fresh checks. + +On `ROLLBACK_FAILED`, retain the lock and all artifacts. Read the named step, +ledger and service journals. Determine which bundle and store the unit selects +before starting it. Resolve the failed operation and complete the documented +recovery sequence manually. Do not delete fsql4, relabel a STORE marker, rerun +the script over partial recovery, or overwrite the preserved bundles. + +## Rehearsal + +See [the rehearsal harness](roll-to-format5-rehearsal/README.md). Its image runs +real systemd as PID 1. It uses isolated copies, generated test identities and no +external network. A passing report includes old/new per-standard counts, file +hashes, exit codes and the ledger for each store and failure case. Docker +measurements describe that environment and do not predict host outage times. + +## Rehearsal barriers + +`SDN_ROLL_REHEARSAL_PAUSE_DIR` enables two barriers in a dedicated, empty +absolute directory: `stopped.waiting` appears after the old service stops, +and `restored.waiting` appears after recovery restores the bundle, before the +old service starts. The rehearsal driver hashes fsql4 at these boundaries +and creates the corresponding `stopped.continue` or `restored.continue` file. +Each barrier has a five-minute limit. A stopped-barrier timeout triggers +recovery; a restored-barrier timeout records `ROLLBACK_FAILED`. Existing +barrier files are refused before the outage. Leave this variable unset in +production; the default has no pause. The directory belongs to the rehearsal +and contains no store or identity files. + +The staged runtime precheck makes at most five attempts, ten seconds apart, +passing on the first success. Every attempt appears in the ledger. All five +failures refuse the rollout before stopping the service. + +The migration repeats its own runtime guard. An exit within the first 30 seconds +with the runtime-patches refusal is retried only when no format-5 candidate +content existed before or after that attempt and no fsql5 selection exists. +There are at most five conversion attempts, ten seconds apart, within one +conversion budget. Every other failure invokes recovery. No runtime check or +threshold is weakened. + +The deployment service uses `Restart=always` and `RestartSec=15`. A daemon +runtime refusal therefore schedules another start after 15 seconds, subject to +systemd's start limits. The readiness loop keeps polling across failed starts +and restarts for the entire ready budget; it does not stop at the first failed +invocation. Check the installed service's restart policy and start limits before +use. The isolated rehearsal uses the same restart settings. + +The self-test gives interpreter spinners two seconds to start and one executor +stop 300 milliseconds to reach every spinner. Its reported AOT speed ratio is +diagnostic and is not the runtime patch pass predicate. diff --git a/deployment/scripts/roll-to-format5.sh b/deployment/scripts/roll-to-format5.sh new file mode 100755 index 0000000000..cb93b5d93a --- /dev/null +++ b/deployment/scripts/roll-to-format5.sh @@ -0,0 +1,341 @@ +#!/usr/bin/env bash +# Offline bundle replacement with a retained format-4 recovery path. +set -euo pipefail +umask 077 + +usage() { + echo 'Usage: roll-to-format5.sh UNIT BUNDLE_BASE STAGED CONFIG STORE READY_URL PEER LEDGER HOLDER TOKEN CONVERT_MINUTES READY_SECONDS [HOME]' >&2 + exit 2 +} +[[ $# == 12 || $# == 13 ]] || usage +unit=$1 base=$2 staged=$3 config=$4 root=$5 ready_url=$6 peer=$7 +ledger=$8 holder=$9 token=${10} minutes=${11} ready_seconds=${12} +node_home=${13:-/root} +[[ $unit != -* && $unit != */* && $unit == *.service ]] || usage +[[ $holder =~ ^[a-zA-Z0-9_.-]+$ && $token =~ ^[a-zA-Z0-9_.-]+$ ]] || usage +[[ $peer =~ ^[a-zA-Z0-9]+$ ]] || usage +[[ $minutes =~ ^[1-9][0-9]{0,4}$ && $ready_seconds =~ ^[1-9][0-9]{0,5}$ ]] || usage +[[ $ready_url == http://* || $ready_url == https://* ]] || usage +[[ $EUID == 0 ]] || { echo 'Root is required.' >&2; exit 2; } +# Canonical paths make rename and containment checks independent of spelling. +for path in "$base" "$staged" "$config" "$root" "$ledger" "$node_home"; do + [[ $path == /* && $path != *$'\n'* ]] || usage +done +base=$(realpath -e -- "$base") +staged=$(realpath -m -- "$staged") +root=$(realpath -e -- "$root") +config=$(realpath -e -- "$config") +live=$base/sdn-bundle +stamp=$(date -u +%Y%m%dT%H%M%S)-$$ +old=$base/sdn-bundle.old-$stamp +failed=$base/sdn-bundle.failed-$stamp +backup=${root}.roll-backup-$stamp +stage=PRECHECK stopped=0 conversion_pid= done_ok=0 +pause_dir=${SDN_ROLL_REHEARSAL_PAUSE_DIR:-} +side_files=(dataset-catalogs dataset-feed-head-sync control.flatsqldb + fsql4-migrate.json source-metrics.db storefront.db peers.db + trusted-import-lanes.json homepage.json) +# Preserve SQLite companions when a stopped database retains them. +for name in source-metrics.db storefront.db peers.db; do + side_files+=("$name-wal" "$name-shm" "$name-journal") +done + +log() { + local line + line="$(date -u +%Y-%m-%dT%H:%M:%SZ) $holder $1 token=$token ${2:-}" + printf '%s\n' "$line" >> "$ledger" + printf '%s\n' "$line" || true +} +fail() { reason=$1; log FAILED "step=$stage reason=$reason"; exit 1; } +exists() { [[ -e $1 || -L $1 ]]; } + +# Explicit rehearsal barriers let the driver hash a quiescent store. +# A missing acknowledgment fails closed after five minutes. +rehearsal_pause() { + [[ -n $pause_dir ]] || return 0 + local point=$1 end=$((SECONDS + 300)) + : > "$pause_dir/$point.waiting" || return 1 + log REHEARSAL_PAUSED "point=$point" || return 1 + while [[ ! -f $pause_dir/$point.continue ]]; do + (( SECONDS < end )) || return 1 + sleep 1 + done +} + +# Parse systemctl's quoted words without evaluating shell code or printing values. +# Unsupported escape spellings fail closed before the unit stops. +unit_env=() +parse_environment() { + local input=$1 word= char quote= escaped=0 i + for ((i=0; i<${#input}; i++)); do + char=${input:i:1} + if (( escaped )); then + case $char in + ' '|\"|\'|\\|'$'|'`') word+=$char ;; + *) return 1 ;; + esac + escaped=0 + elif [[ $char == \\ ]]; then + escaped=1 + elif [[ -n $quote ]]; then + if [[ $char == "$quote" ]]; then quote=; else word+=$char; fi + elif [[ $char == '"' || $char == "'" ]]; then + quote=$char + elif [[ $char == ' ' ]]; then + if [[ -n $word ]]; then unit_env+=("$word"); word=; fi + else + word+=$char + fi + done + [[ -z $quote && $escaped == 0 ]] || return 1 + [[ -z $word ]] || unit_env+=("$word") + for word in "${unit_env[@]}"; do + [[ $word =~ ^[a-zA-Z_][a-zA-Z0-9_]*= ]] || return 1 + done +} +stop_unit() { + local end=$((SECONDS + 90)) pid state + systemctl stop --no-block "$unit" || return 1 + while (( SECONDS < end )); do + pid=$(systemctl show -p MainPID --value "$unit") || return 1 + state=$(systemctl show -p ActiveState --value "$unit") || return 1 + if [[ $pid == 0 && ( $state == inactive || $state == failed ) ]]; then return 0; fi + sleep 1 + done + return 1 +} +wait_ready() { + local end=$((SECONDS + ready_seconds)) status left + while (( SECONDS < end )); do + left=$((end - SECONDS)) + (( left <= 5 )) || left=5 + status=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --max-time "$left" -- "$ready_url") || status=000 + if [[ $status == 200 ]] && systemctl is-active --quiet "$unit"; then return 0; fi + sleep 1 + done + return 1 +} +check_running_peer() { + local invocation lines line found= + invocation=$(systemctl show -p InvocationID --value "$unit") || return 1 + [[ $invocation =~ ^[a-f0-9]{32}$ ]] || return 1 + lines=$(journalctl --no-pager -o cat "_SYSTEMD_INVOCATION_ID=$invocation" \ + --grep='Peer ID: ') || return 1 + while IFS= read -r line; do + if [[ $line =~ Peer\ ID:\ ([a-zA-Z0-9]+) ]]; then found=${BASH_REMATCH[1]}; fi + done <<< "$lines" + [[ $found == "$peer" ]] +} +stop_conversion() { + [[ -n $conversion_pid ]] || return 0 + if kill -0 "$conversion_pid" 2>/dev/null; then + # Inspect only this child's command line before signaling its exact PID. + local arg matched=0 has_args=0 + while IFS= read -r -d '' arg; do + has_args=1 + [[ $arg != store-migrate ]] || matched=1 + done < "/proc/$conversion_pid/cmdline" + if (( matched )); then + kill -KILL "$conversion_pid" 2>/dev/null || ! kill -0 "$conversion_pid" 2>/dev/null || return 1 + elif (( has_args )); then + return 1 + fi + fi + wait "$conversion_pid" 2>/dev/null || true + conversion_pid= +} +rollback() { + local after=0 name + stage=STOP_CONVERSION + stop_conversion || return 1 + exists "$root/fsql5" && after=1 + log ROLLBACK "after_switch=$after" || return 1 + if exists "$old" || (( after )); then + stage=STOP_FOR_ROLLBACK + stop_unit || return 1 + fi + if (( after )); then + stage=RESTORE_STORE + [[ -d $backup && -f $root/fsql4/rollback-auxiliary.flatsqlmeta ]] || return 1 + mv -T -- "$root/fsql5" "$root/fsql5.failed-$stamp" || return 1 + cp -p -- "$root/fsql4/rollback-auxiliary.flatsqlmeta" "$root/auxiliary.flatsqlmeta" || return 1 + stage=RESTORE_SIDE_FILES + mkdir -- "$backup/changed" || return 1 + for name in "${side_files[@]}"; do + if exists "$root/$name"; then + mv -T -- "$root/$name" "$backup/changed/$name" || return 1 + fi + if exists "$backup/$name"; then + cp -a -- "$backup/$name" "$root/$name" || return 1 + fi + done + fi + stage=RESTORE_BUNDLE + if exists "$old"; then + if exists "$live"; then mv -T -- "$live" "$failed" || return 1; fi + mv -T -- "$old" "$live" || return 1 + fi + stage=REHEARSAL_RESTORED + rehearsal_pause restored || return 1 + stage=START_OLD + systemctl start --no-block "$unit" || return 1 + wait_ready || return 1 + stage=CHECK_OLD_IDENTITY + check_running_peer || return 1 + if (( after )); then + log ROLLED_BACK_AFTER_SWITCH "reason=$reason" || return 1 + else + log ROLLED_BACK_BEFORE_SWITCH "reason=$reason" || return 1 + fi +} +finish() { + local status=$? failed_step=$stage + trap - EXIT + trap '' TERM HUP INT PIPE + (( done_ok )) && exit 0 + reason=${reason:-"step=$failed_step exit=$status"} + if (( stopped )); then + if ! rollback; then + log ROLLBACK_FAILED "step=$stage reason=$reason" || true + fi + else + log PRECHECK_FAILED "$reason" || true + fi + exit 1 +} +trap finish EXIT +trap 'reason=SIGTERM; exit 143' TERM +trap 'reason=SIGHUP; exit 129' HUP +trap 'reason=SIGINT; exit 130' INT + +log PRECHECK 'begin' +if [[ -n $pause_dir ]]; then + [[ $pause_dir == /* && -d $pause_dir && ! -L $pause_dir ]] || fail rehearsal_pause_directory + for point in stopped restored; do + ! exists "$pause_dir/$point.waiting" && ! exists "$pause_dir/$point.continue" || fail rehearsal_pause_stale + done +fi +[[ $(systemctl show -p LoadState --value "$unit") == loaded ]] || fail unit_missing +systemctl is-active --quiet "$unit" || fail unit_inactive +user=$(systemctl show -p User --value "$unit") +[[ -z $user || $user == root ]] || fail unit_not_root +[[ -z $(systemctl show -p EnvironmentFiles --value "$unit") ]] || fail environment_files_not_supported +parse_environment "$(systemctl show -p Environment --value "$unit")" || fail environment_encoding +home_override=() +[[ $# == 12 ]] || home_override=("HOME=$node_home") +node_command=(env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin HOME="$node_home" "${unit_env[@]}" "${home_override[@]}") +[[ -d $live && ! -L $live && -d $staged && -f $config ]] || fail bundle_or_config_missing +[[ $staged != "$live" && $staged != "$live/"* && $live != "$staged/"* ]] || fail overlapping_bundles +[[ $root != "$base" && $root != "$base/"* && $base != "$root/"* ]] || fail overlapping_store +[[ $root != "$staged" && $root != "$staged/"* && $staged != "$root/"* ]] || fail overlapping_staging +[[ $(stat -c %d "$staged") == "$(stat -c %d "$base")" ]] || fail different_filesystems +for path in "$old" "$failed" "$backup" "$root/fsql5.failed-$stamp"; do + ! exists "$path" || fail recovery_path_exists +done +[[ -d $root/fsql4 && ! -L $root/fsql4 ]] || fail format4_missing +! exists "$root/fsql5" || fail format5_exists +[[ -f $root/auxiliary.flatsqlmeta && ! -L $root/auxiliary.flatsqlmeta ]] || fail auxiliary_missing +"${node_command[@]}" "$staged/bin/spacedatanetwork" version >/dev/null 2>&1 || fail staged_version +identity=$("${node_command[@]}" "$staged/bin/spacedatanetwork" show-identity -c "$config" 2>/dev/null) || fail identity_unreadable +[[ $identity == "$peer" ]] || fail identity_mismatch +check_running_peer || fail running_identity_mismatch +bytes=$(du -sb -- "$root/fsql4"); bytes=${bytes%%$'\t'*} +aux=$(stat -c %s -- "$root/auxiliary.flatsqlmeta") +free=$(df -B1 --output=avail -- "$root" | tail -n 1); free=${free// /} +need=$((3 * bytes + aux + 1073741824 + 5000000000)) +log DISK "required_bytes=$need available_bytes=$free" +(( free >= need )) || fail disk_gate +runtime_ok=0 +for ((attempt=1; attempt<=5; attempt++)); do + log RUNTIME_CHECK "attempt=$attempt" + if "${node_command[@]}" "$staged/bin/spacedatanetwork" substrate-selftest --require-patched; then + runtime_ok=1 + log RUNTIME_CHECK_OK "attempt=$attempt" + break + fi + log RUNTIME_CHECK_REFUSED "attempt=$attempt" + (( attempt == 5 )) || sleep 10 +done +if (( ! runtime_ok )); then + log RUNTIME_CHECK_FAILED 'attempts=5' + fail staged_runtime +fi +log PRECHECK_OK +stage=STOP +stopped=1 +stop_unit || fail stop_timeout +log STOPPED +stage=REHEARSAL_STOPPED +rehearsal_pause stopped || fail rehearsal_pause_timeout +stage=SWAP +mv -T -- "$live" "$old" +mv -T -- "$staged" "$live" +log SWAPPED +stage=BACKUP +mkdir -- "$backup" +for name in "${side_files[@]}"; do + if exists "$root/$name"; then cp -a -- "$root/$name" "$backup/$name"; fi +done +log BACKED_UP +stage=CONVERT +# Retry only an early runtime refusal with no candidate content. +format5_empty() ( + shopt -s nullglob dotglob + ! exists "$root/fsql5" || exit 1 + entries=("$root/.store-format5-pending"/*) + (( ${#entries[@]} == 0 )) +) +deadline=$((SECONDS + minutes * 60)) +for ((attempt=1; attempt<=5; attempt++)); do + (( SECONDS < deadline )) || fail conversion_budget + empty_before=0 + format5_empty && empty_before=1 + attempt_log=$backup/conversion-attempt-$attempt.log + attempt_start=$SECONDS + # A child lets the shell handle signals while it supervises the task. + signal_pending= + trap 'signal_pending=SIGTERM' TERM + trap 'signal_pending=SIGHUP' HUP + trap 'signal_pending=SIGINT' INT + "${node_command[@]}" "$live/bin/spacedatanetwork" store-migrate --to 5 --config "$config" --store "$root" >"$attempt_log" 2>&1 & + conversion_pid=$! + trap 'reason=SIGTERM; exit 143' TERM + trap 'reason=SIGHUP; exit 129' HUP + trap 'reason=SIGINT; exit 130' INT + if [[ -n $signal_pending ]]; then reason=$signal_pending; exit 1; fi + log CONVERTING "pid=$conversion_pid budget_minutes=$minutes attempt=$attempt output=$attempt_log" + while kill -0 "$conversion_pid" 2>/dev/null; do + (( SECONDS < deadline )) || fail conversion_budget + sleep 1 + done + status=0 + wait "$conversion_pid" || status=$? + conversion_pid= + elapsed=$((SECONDS - attempt_start)) + cat -- "$attempt_log" + (( status != 0 )) || break + refused=0 + while IFS= read -r line; do + [[ $line != *'store-migrate --to 5 cannot run: the linked WasmEdge '*'lacks the SDN runtime patches'* ]] || refused=1 + done < "$attempt_log" + if (( attempt == 5 || ! refused || ! empty_before || elapsed >= 30 )) || ! format5_empty; then + fail conversion_exit + fi + log CONVERSION_RETRY "attempt=$attempt exit=$status elapsed_seconds=$elapsed reason=runtime_refused" + for ((delay=0; delay<10; delay++)); do + (( SECONDS < deadline )) || fail conversion_budget + sleep 1 + done +done +stage=CHECK_SWITCH +marker=$(od -An -tx1 -j4 -N2 -- "$root/fsql5/STORE"); marker=${marker// /} +[[ $marker == 0500 && -d $root/fsql4 && -f $root/fsql4/rollback-auxiliary.flatsqlmeta ]] || fail invalid_switch +log SWITCHED +stage=START_NEW +systemctl start --no-block "$unit" +wait_ready || fail ready_budget +stage=CHECK_NEW_IDENTITY +check_running_peer || fail running_identity_mismatch +log SUCCESS 'format=5' +done_ok=1